Skip to content

Commit

Permalink
Merge pull request 'Release v24.12' (#4) from release_24.12 into master
Browse files Browse the repository at this point in the history
  • Loading branch information
Dominick Leppich committed Jan 13, 2025
2 parents d92a760 + f0b5c61 commit 706348c
Show file tree
Hide file tree
Showing 13 changed files with 245 additions and 38 deletions.
3 changes: 1 addition & 2 deletions docs/index_de.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,5 @@ Die folgende Tabelle enthält eine Zusammenstellung der Parameter und ihrer Besc
Parameter | Erläuterung
------------------------|------------------------------------
`title` | Hier kann ein individueller Titel für die Anzeige im Menü und die Überschriften des Plugins festgelegt werden.
`importSet` | Mit diesem Element lassen sich individuelle Importsets definieren. Sie bestehen jeweils aus einem `title` für den anzuzeigenden Namen, sowie einer Angabe für den Speicherort, von dem die Daten importiert werden sollen. Der Speicherort kann dabei beliebig tiefe Verzeichnisstrukturen aufweisen.
`workflow` | Hiermit läßt sich festlegen, welche Produktionsvorlage für das Anlegen von Goobi-Vorgängen verwendet werden soll.
`importSet` | Mit diesem Element lassen sich individuelle Importsets definieren. Sie bestehen jeweils aus einem `title` für den anzuzeigenden Namen, sowie einer Angabe für den Speicherort, von dem die Daten importiert werden sollen. Der Speicherort kann dabei beliebig tiefe Verzeichnisstrukturen aufweisen. Ausserdem läßt sich mit `workflow` festlegen, welche Produktionsvorlage für das Anlegen von Goobi-Vorgängen verwendet werden soll.

3 changes: 1 addition & 2 deletions docs/index_en.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,4 @@ The following table contains a summary of the parameters and their descriptions:
Parameter | Explanation
------------------------|------------------------------------
`title` | An individual title for the display in the menu and the headings of the plugin can be defined here.
`importSet` | Individual import sets can be defined with this element. They each consist of a `title` for the name to be displayed and a specification for the storage location from which the data is to be imported. The storage location can have any depth of directory structure.
`workflow` | This allows you to specify which production template is to be used for creating Goobi processes.
`importSet` | Individual import sets can be defined with this element. They each consist of a `title` for the name to be displayed and a specification for the storage location from which the data is to be imported. The storage location can have any depth of directory structure. You can also use `workflow` to specify which production template is to be used for creating Goobi processes.
13 changes: 5 additions & 8 deletions install/plugin_intranda_workflow_wu_wma_import.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,9 @@
<title>Mass import for the WU</title>

<!-- importsets with source folder definition -->
<importSet title="Markenstudien 1" source="/opt/digiverso/import/markenstudien01/" />
<importSet title="Markenstudien 2" source="/opt/digiverso/import/markenstudien02/" />
<importSet title="Markenstudien 3" source="/opt/digiverso/import/markenstudien03/" />
<importSet title="Markenstudien 4" source="/opt/digiverso/import/markenstudien04/" />

<!-- which workflow to use -->
<workflow>Sample_Workflow</workflow>

<importSet title="Markenstudien 1" workflow="Marken-Workflow" source="/opt/digiverso/import/markenstudien01/" />
<importSet title="Markenstudien 2" workflow="Marken-Workflow" source="/opt/digiverso/import/markenstudien02/" />
<importSet title="Markenstudien 3" workflow="Marken-Workflow" source="/opt/digiverso/import/markenstudien03/" />
<importSet title="Markenstudien 4" workflow="Marken-Workflow" source="/opt/digiverso/import/markenstudien04/" />

</config_plugin>
2 changes: 1 addition & 1 deletion module-base/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<parent>
<groupId>io.goobi.workflow.plugin</groupId>
<artifactId>plugin-workflow-wu-wma-import</artifactId>
<version>24.11</version>
<version>24.12</version>
</parent>
<artifactId>plugin-workflow-wu-wma-import-base</artifactId>
<packaging>jar</packaging>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.apache.commons.collections4.queue.CircularFifoQueue;
import org.apache.commons.configuration.HierarchicalConfiguration;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.goobi.beans.JournalEntry;
import org.goobi.beans.JournalEntry.EntryType;
import org.goobi.beans.Process;
Expand Down Expand Up @@ -83,8 +84,8 @@ public class WuWmaImportWorkflowPlugin implements IWorkflowPlugin, IPushPlugin {
int itemsTotal = 0;
@Getter
private Queue<LogMessage> logQueue = new CircularFifoQueue<LogMessage>(50000);

private String workflow;
@Getter
private int errors = 0;

@Override
public PluginType getType() {
Expand Down Expand Up @@ -112,19 +113,19 @@ public WuWmaImportWorkflowPlugin() {
private void readConfiguration() {
updateLog("Start reading the configuration");

errors = 0;

// set specific title
title = ConfigPlugins.getPluginConfig(id).getString("title");

// read some main configuration
workflow = ConfigPlugins.getPluginConfig(id).getString("workflow");

// read list of mapping configuration
importSets = new ArrayList<ImportSet>();
List<HierarchicalConfiguration> mappings = ConfigPlugins.getPluginConfig(id).configurationsAt("importSet");
for (HierarchicalConfiguration node : mappings) {
String settitle = node.getString("[@title]", "-");
String source = node.getString("[@source]", "-");
importSets.add(new ImportSet(settitle, source));
String workflow = node.getString("[@workflow]", "-");
importSets.add(new ImportSet(settitle, source, workflow));
}

// write a log into the UI
Expand Down Expand Up @@ -175,7 +176,7 @@ public void startImport(ImportSet importset) {
updateLog("Start importing: Record " + (itemCurrent + 1), 1);

// get the correct workflow to use
Process template = ProcessManager.getProcessByExactTitle(workflow);
Process template = ProcessManager.getProcessByExactTitle(importset.getWorkflow());
Prefs prefs = template.getRegelsatz().getPreferences();

// create digital document
Expand Down Expand Up @@ -251,6 +252,16 @@ public void startImport(ImportSet importset) {
for (SimpleContent con : sio.getProcess().getContents()) {
String targetBase = process.getConfiguredImageFolder(con.getFolder().trim());

// get the source folder
if (targetBase == null && "source".equals(con.getFolder().trim())) {
targetBase = process.getSourceDirectory();
}

// get the import folder
if (targetBase == null && "import".equals(con.getFolder().trim())) {
targetBase = process.getImportDirectory();
}

// if the target folder cannot be found
if (targetBase == null) {
updateLog("Error: Target folder '" + con.getFolder() + "' does not exist for file '" + con.getSource() + "'", 3);
Expand All @@ -274,9 +285,11 @@ public void startImport(ImportSet importset) {
updateLog("Process successfully created with ID: " + process.getId());

} catch (Exception e) {
log.error("Error while creating a process during the import", e);
updateLog("Error while creating a process during the import: " + e.getMessage(), 3);
Helper.setFehlerMeldung("Error while creating a process during the import: " + e.getMessage());
log.error("Error while creating a process during the import for file " + file.getAbsolutePath(), e);
updateLog("Error while creating a process during the import for file " + file.getAbsolutePath() + ": " + e.getMessage(),
3);
Helper.setFehlerMeldung(
"Error while creating a process during the import for file " + file.getAbsolutePath() + ": " + e.getMessage());
pusher.send("error");
}

Expand Down Expand Up @@ -311,9 +324,9 @@ public void startImport(ImportSet importset) {
* @throws MetadataTypeNotAllowedException
*/
private Corporate getCorporate(Prefs prefs, SimpleCorporate sc) throws MetadataTypeNotAllowedException {
MetadataType mdt = prefs.getMetadataTypeByName(sc.getRole());
MetadataType mdt = prefs.getMetadataTypeByName(sc.getRole().trim());
if (mdt == null) {
updateLog("The metadata type with name '" + sc.getRole()
updateLog("The metadata type with name '" + sc.getRole().trim()
+ "' does not exist within the ruleset.", 3);
}
Corporate c = new Corporate(mdt);
Expand All @@ -337,13 +350,17 @@ private Corporate getCorporate(Prefs prefs, SimpleCorporate sc) throws MetadataT
* @throws MetadataTypeNotAllowedException
*/
private Metadata getMetadata(Prefs prefs, SimpleMetadata sm) throws MetadataTypeNotAllowedException {
MetadataType mdt = prefs.getMetadataTypeByName(sm.getType());
MetadataType mdt = prefs.getMetadataTypeByName(sm.getType().trim());
if (mdt == null) {
updateLog("The metadata type with name '" + sm.getType()
updateLog("The metadata type with name '" + sm.getType().trim()
+ "' does not exist within the ruleset.", 3);
}
if (StringUtils.isBlank(sm.getValue())) {
updateLog("The metadata of type '" + sm.getType().trim()
+ "' is empty.", 3);
}
Metadata m = new Metadata(mdt);
m.setValue(sm.getValue());
m.setValue(sm.getValue().trim());
m.setAuthorityFile(sm.getAuthority(), sm.getAuthorityURI(), sm.getValueURI());
return m;
}
Expand All @@ -357,9 +374,9 @@ private Metadata getMetadata(Prefs prefs, SimpleMetadata sm) throws MetadataType
* @throws MetadataTypeNotAllowedException
*/
private Person getPerson(Prefs prefs, SimplePerson sp) throws MetadataTypeNotAllowedException {
MetadataType mdt = prefs.getMetadataTypeByName(sp.getRole());
MetadataType mdt = prefs.getMetadataTypeByName(sp.getRole().trim());
if (mdt == null) {
updateLog("The metadata type with name '" + sp.getRole()
updateLog("The metadata type with name '" + sp.getRole().trim()
+ "' does not exist within the ruleset.", 3);
}
Person p = new Person(mdt);
Expand All @@ -378,9 +395,9 @@ private Person getPerson(Prefs prefs, SimplePerson sp) throws MetadataTypeNotAll
* @throws MetadataTypeNotAllowedException
*/
private MetadataGroup getGroup(Prefs prefs, SimpleGroup sg) throws MetadataTypeNotAllowedException {
MetadataGroupType mdt = prefs.getMetadataGroupTypeByName(sg.getType());
MetadataGroupType mdt = prefs.getMetadataGroupTypeByName(sg.getType().trim());
if (mdt == null) {
updateLog("The metadata group type with name '" + sg.getType()
updateLog("The metadata group type with name '" + sg.getType().trim()
+ "' does not exist within the ruleset.", 3);
}
MetadataGroup group = new MetadataGroup(mdt);
Expand All @@ -391,9 +408,9 @@ private MetadataGroup getGroup(Prefs prefs, SimpleGroup sg) throws MetadataTypeN
}

// create all corporates
// for (SimpleCorporate sc : sg.getCorporates()) {
// group.addCorporate(getCorporate(prefs, sc));
// }
for (SimpleCorporate sc : sg.getCorporates()) {
group.addCorporate(getCorporate(prefs, sc));
}

// create all metadata groups
for (SimpleGroup ssg : sg.getGroups()) {
Expand Down Expand Up @@ -428,6 +445,10 @@ private void updateLog(String logmessage) {
* @param logmessage
*/
private void updateLog(String logmessage, int level) {
if (level > 1) {
errors++;
}

logQueue.add(new LogMessage(logmessage, level));
log.debug(logmessage);
if (pusher != null && System.currentTimeMillis() - lastPush > 500) {
Expand All @@ -441,6 +462,7 @@ private void updateLog(String logmessage, int level) {
public class ImportSet {
private String title;
private String source;
private String workflow;
}

@Data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;

Expand All @@ -13,6 +15,9 @@
@Setter
public class SimpleCorporate {

@JacksonXmlProperty(isAttribute = true)
private String label;

@JacksonXmlProperty(localName = "subname")
@JacksonXmlElementWrapper(useWrapping = false)
private List<String> subnames = new ArrayList<String>();
Expand All @@ -34,4 +39,25 @@ public class SimpleCorporate {
@JacksonXmlProperty(isAttribute = true)
private String name;

/**
* add authority information based on just one url
*
* @param authorityEntryUrl
*/
public void analyzeAuthority(String authorityEntryUrl) {
authority = null;
authorityURI = null;
valueURI = null;

// split authority information
if (StringUtils.isNotBlank(authorityEntryUrl)) {
if (authorityEntryUrl.contains("d-nb.info")) {
authority = "gnd";
}
if (authorityEntryUrl.contains("/")) {
authorityURI = authorityEntryUrl.substring(0, authorityEntryUrl.lastIndexOf("/") + 1);
valueURI = authorityEntryUrl.substring(authorityEntryUrl.lastIndexOf("/") + 1);
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;

Expand Down Expand Up @@ -31,4 +33,46 @@ public class SimpleData {

@JacksonXmlProperty(isAttribute = true)
private String type;

/**
* Add a {@link SimpleMetadata}
*
* @param md
*/
public void add(SimpleMetadata md) {
if (StringUtils.isNoneBlank(md.getValue(), md.getType())) {
metadatas.add(md);
}
}

/**
* add a {@link SimplePerson}
*
* @param p
*/
public void add(SimplePerson p) {
if (StringUtils.isNoneBlank(p.getRole(), p.getLastname())) {
persons.add(p);
}
}

/**
* add a {@link SimpleCorporate}
*
* @param c
*/
public void add(SimpleCorporate c) {
if (StringUtils.isNoneBlank(c.getName(), c.getRole())) {
corporates.add(c);
}
}

/**
* add a {@link SimpleGroup}
*
* @param g
*/
public void add(SimpleGroup g) {
groups.add(g);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;

Expand Down Expand Up @@ -35,4 +37,45 @@ public class SimpleGroup {
@JacksonXmlProperty(localName = "group")
private List<SimpleGroup> groups = new ArrayList<SimpleGroup>();

/**
* Add a {@link SimpleMetadata}
*
* @param md
*/
public void add(SimpleMetadata md) {
if (StringUtils.isNoneBlank(md.getValue(), md.getType())) {
metadatas.add(md);
}
}

/**
* add a {@link SimplePerson}
*
* @param p
*/
public void add(SimplePerson p) {
if (StringUtils.isNoneBlank(p.getRole(), p.getLastname())) {
persons.add(p);
}
}

/**
* add a {@link SimpleCorporate}
*
* @param c
*/
public void add(SimpleCorporate c) {
if (StringUtils.isNoneBlank(c.getName(), c.getRole())) {
corporates.add(c);
}
}

/**
* add a {@link SimpleGroup}
*
* @param g
*/
public void add(SimpleGroup g) {
groups.add(g);
}
}
Loading

0 comments on commit 706348c

Please sign in to comment.