diff --git a/LICENCE.txt b/LICENCE.txt new file mode 100644 index 0000000..54fec5d --- /dev/null +++ b/LICENCE.txt @@ -0,0 +1,23 @@ +The MIT License + +Copyright (c) 2012, University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + diff --git a/README.txt b/README.txt index 96c65d7..3db4301 100644 --- a/README.txt +++ b/README.txt @@ -8,118 +8,11 @@ RDF-enhanced storage API, pairtree-backed. Installation and dependancies ============================= -- [I would advise using virtualenv] +The dependancies are listed in the file docs/Dependancies.pdf +The installation instructions are available at docs/Databank_VM_Installation.txt -Dependancies: +For further infromation visit http://www.dataflow.ox.ac.uk/ +The project wiki is available at http://dataflow-jira.bodleian.ox.ac.uk/jira/ or +https://damssupport.ouls.ox.ac.uk/trac/databank/wiki/ -pylons==0.9.7 -pairtree==0.5.6-T -rdfobject -recordsilo -simplejson -solrpy - -redis (use the redis client that is compatible with your redis server - http://github.com/andymccurdy/redis-py/) - - -- Change the settings in development.ini to suit your uses, specifically: - -redis.host = localhost - -granary.store = silos # Directory to storage directory. Can be absolute: /var/silos -granary.uri_root = http://databank.bodleian.ox.ac.uk/objects/ # URI root - must end with /, # or : - -broadcast.to = redis -broadcast.queue = silochanges # redis 'queue' to push audit messages to (create, delete, update or embargo change) - -solr.host = http://localhost:8983/solr # Unset this to ignore solr while testing - -Setup: -====== - -Create a file 'passwd' in the root directory of the application using 'htpasswd' or similar: - -$ htpasswd -c passwd admin -[enter admin password] - -Add any users you wish to access or work with this application. - -Edit ./rdfdatabank/lib/ident_md.py - this has a variable _DATA which would best be replaced by a DB lookup or similar. For now, adjust it -to suit your users - the important thing is the 'role'. - -Start up the redis-server and confirm that you can access it. - -You should be able to start the application now (as long as the application has access to r+w to the aforementioned 'silos' directory.) - -paster serve development.ini - -Then, go to localhost:5000/admin, and try to log in as the 'admin' account. You should then be able to create and manage silos and assign write-access to them to users. - -WGSI deployment: -================ - -Copy 'development.ini' to a file called 'production.ini' and make sure debug = false and that the 'silos' directory is -owned by the web-server user (www-data for example). Recommendation is to have a separate silo directory for production use. - -Create a folder called 'egg-cache' and make sure that is writable by the web-server user too. - -Create a folder 'mod_wsgi' in the root of the application. - -Create a file within that called 'dispatch.wsgi' and put in the following: - ---------------- -# Add the virtual Python environment site-packages directory to the path - -import pkg_resources -pkg_resources.working_set.add_entry('/opt/rdfdatabank/src') # Path to application root - -# Avoid ``[Errno 13] Permission denied: '/var/www/.python-eggs'`` messages -import os -os.environ['PYTHON_EGG_CACHE'] = '/opt/rdfdatabank/src/egg-cache' # Path to writable directory for egg-cache - -# Load the Pylons application -from paste.deploy import loadapp -application = loadapp('config:/opt/rdfdatabank/src/production.ini') # Path to production.ini file - ---------------- - -In your apache2 configuration, you need an entry like: - - - ServerName foo.com - - # Logfiles - ErrorLog /opt/rdfdatabank/apachelog/error.log - CustomLog /opt/rdfdatabank/apachelog/access.log combined - - # Use only 1 Python sub-interpreter. Multiple sub-interpreters - # play badly with C extensions. - WSGIApplicationGroup %{GLOBAL} - WSGIPassAuthorization On - # Setup mod_wsgi - WSGIScriptAlias / /opt/rdfdatabank/src/mod_wsgi/dispatch.wsgi - - - Order deny,allow - Allow from all - - - - -Note the WSGIScriptAlias points to the dispatch.wsgi file. - ---------------- - -Note about permissions: -src directory and all its files and directories: - Set the owner as admin_user:www-data - Set the permission to 775 on all directories and everything under it - set the owner as www-data:www-data for src/data folder - Set the permission to 644 on all files -apachelog directory and all its files and directories: - Set the owner as admin_user:www-data - Set the persmissions 755 (currently the dir permission is set to 755 and the permission of the files within it is set to 644) -All of other directories: - Set the owner as admin_user:admin_user - Set the permission to 775 on all directories and everything under it diff --git a/debian/README b/debian/README new file mode 100644 index 0000000..433bddc --- /dev/null +++ b/debian/README @@ -0,0 +1,6 @@ +The Debian Package databank +---------------------------- + +Comments regarding the Package + + -- Anusha Ranganathan Fri, 09 Mar 2012 17:08:50 +0000 diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..8e38f94 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,13 @@ +databank (0.2) unstable; urgency=low + + * Added message workers + * Databank root set to /var/lib/databank + + -- Anusha Ranganathan Fri, 09 Mar 2012 17:08:50 +0000 + + +databank (0.1) unstable; urgency=low + + * Initial Release. + + -- Anusha Ranganathan Tue, 21 Feb 2012 17:08:50 +0000 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..7f8f011 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +7 diff --git a/debian/config b/debian/config new file mode 100644 index 0000000..0bdeff2 --- /dev/null +++ b/debian/config @@ -0,0 +1,37 @@ +#!/bin/sh -e + +# Source debconf library. +. /usr/share/debconf/confmodule + +# Admin email? +db_input critical databank/email || true +db_go + +# SMTP server? +db_input high databank/smtpserver || true +db_go + +# Storgae area? +db_input high databank/store || true +db_go + +# Databank URI? +db_input critical databank/uri || true +db_go + +#Configure SOLR +db_input high databank/confsolr || true +db_go + +#Note on configuring SOLR +#db_input high databank/nosolrconf || true +#db_go + +# Password? +#db_input critical databank/password || true +#db_go + +# Note on Password +db_input critical databank/passwdnote || true +db_go + diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..44ee2a6 --- /dev/null +++ b/debian/control @@ -0,0 +1,37 @@ +Source: databank +Section: web +Priority: extra +Maintainer: Anusha Ranganathan +Build-Depends: debhelper (>= 7) +Standards-Version: 3.9.2 +Vcs-Git: git://github.com/dataflow/RDFDatabank.git +Homepage: https://github.com/dataflow/RDFDatabank +Uploaders: Anusha Ranganathan +XS-Python-Version: >= 2.6 + +Package: databank +Architecture: all +Depends: python, + python-pylons, + python-repoze.who, + python-repoze.who-friendlyform, + python-rdflib, + python-dateutil, + python-libxml2, + python-libxslt1, + python-pairtree, + python-recordsilo, + python-solrpy, + python-redis, + unzip, + apache2, + apache2-utils, + libapache2-mod-wsgi, + redis-server, + supervisor, + debconf, + openjdk-6-jre +Recommends: solr-tomcat +Description: RDF-enhanced, pairtree-backed storage API + Databank is a simple, RDF-enhanced storage API which is backed by pairtree, + for the safe storage of and access to data. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..b0264d8 --- /dev/null +++ b/debian/copyright @@ -0,0 +1,42 @@ +This work was packaged for Debian by: + + Anusha Ranganathan on Fri, 09 Mar 2012 17:08:50 +0000 + +It was downloaded from: + + https://github.com/dataflow/RDFDatabank + +Upstream Author(s): + + Anusha Ranganathan + Ben O'Steen + +Copyright: + + Copyright (C) 2012 University of Oxford + +License: + + MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +The Debian packaging is: + + Copyright (C) 2012 University of Oxford + and is licensed under the MIT License. See above. diff --git a/debian/databank.install b/debian/databank.install new file mode 100644 index 0000000..7096838 --- /dev/null +++ b/debian/databank.install @@ -0,0 +1,12 @@ +docs/apache_config/databank_wsgi /etc/default/databank/ +docs/solr_config/conf/schema.xml /etc/default/databank/ +production.ini /etc/default/databank/ +development.ini /etc/default/databank/ +MANIFEST.in /var/lib/databank/ +mod_wsgi /var/lib/databank/ +message_workers/ /var/lib/databank/ +passwd-default /var/lib/databank/ +rdfdatabank/ /var/lib/databank/ +rdfdatabank.egg-info/ /var/lib/databank +test.ini /var/lib/databank/ +who.ini /var/lib/databank/ diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..eb96eff --- /dev/null +++ b/debian/docs @@ -0,0 +1,7 @@ +LICENCE.txt +README.txt +requirements.txt +docs/assigning_dois +docs/Databank_VM_Installation.txt +docs/Dependencies.pdf +docs/using_databank_api diff --git a/debian/postinst b/debian/postinst new file mode 100644 index 0000000..187b6b0 --- /dev/null +++ b/debian/postinst @@ -0,0 +1,135 @@ +#!/bin/sh +# postinst script for databank +# +# see: dh_installdeb(1) + +set -e + +# Source debconf library. +. /usr/share/debconf/confmodule + +# Ask questions +db_get databank/email +local dbemail="$RET" +#echo "Email id is:" $dbemail + +db_get databank/smtpserver +local dbsmtp="$RET" + +db_get databank/store +local dbstore="$RET" + +db_get databank/uri +local dburi="$RET" + +#db_get databank/password +#local dbpasswd="$RET" + +#Modify the ini files +CONFIG_FILE1=/etc/default/databank/production.ini +CONFIG_FILE2=/etc/default/databank/development.ini +#replace email +TARGET_KEY=email_to +echo "Setting email Id: " $dbemail +#Sed syntax: sed s/a/b/ or sed s,a,b, +sed -i "s/\($TARGET_KEY *= *\).*/\1$dbemail/" $CONFIG_FILE1 +sed -i "s/\($TARGET_KEY *= *\).*/\1$dbemail/" $CONFIG_FILE2 +#replace smtp server +echo "Setting SMTP Server: " $dbsmtp +TARGET_KEY=smtp_server +sed -i "s,\($TARGET_KEY *= *\).*,\1$dbsmtp," $CONFIG_FILE1 +sed -i "s,\($TARGET_KEY *= *\).*,\1$dbsmtp," $CONFIG_FILE2 +#replace granary store +echo "Setting Data store: " $dbstore +TARGET_KEY=granary.store +sed -i "s,\($TARGET_KEY *= *\).*,\1$dbstore," $CONFIG_FILE1 +sed -i "s,\($TARGET_KEY *= *\).*,\1$dbstore," $CONFIG_FILE2 +#replace granary uri root +echo "Setting Base URI: " $dburi +TARGET_KEY=granary.uri_root +[ $TARGET_KEY != "*/" ] && TARGET_KEY="$TARGET_KEY""/" +sed -i "s,\($TARGET_KEY *= *\).*,\1$dburi," $CONFIG_FILE1 +sed -i "s,\($TARGET_KEY *= *\).*,\1$dburi," $CONFIG_FILE2 + +#Copy htpasswd file +#htpasswd -b -c /var/lib/databank/passwd admin $dbpasswd +cp /var/lib/databank/passwd-default /var/lib/databank/passwd + +#Link config files +ln -sf /etc/default/databank/production.ini /var/lib/databank/production.ini +ln -sf /etc/default/databank/development.ini /var/lib/databank/development.ini +ln -sf /etc/default/databank/databank_wsgi /etc/apache2/sites-available/databank_wsgi + +#Create dirs and change ownership and permissions +# ownership and permissions for /var/lib/databank +chgrp -R www-data /var/lib/databank/ +chmod -R 770 /var/lib/databank/ +# ownership and permissions for /var/log/databank +mkdir -p /var/log/databank +chgrp -R www-data /var/log/databank/ +chmod -R 770 /var/log/databank/ +# ownership and permissions for /var/cache/databank +mkdir -p /var/cache/databank +chgrp -R www-data /var/cache/databank/ +chmod -R 770 /var/cache/databank/ +# ownership and permissions /etc/default/databank +chgrp -R www-data /etc/default/databank/ +chmod -R 770 /var/cache/databank/ +# ownership and permissions granary.store +mkdir -p $dbstore +chgrp -R www-data $dbstore +chmod -R 770 $dbstore + +#Configure SOLR and message workers in supervisor +db_get databank/confsolr +local dbconf="$RET" +installstatus=`dpkg-query -W -f='${Status}' solr-tomcat` +if [ "$installstatus" = "install ok installed" ] +then + if [ "$dbconf" = "true" ] + then + if [ -f /etc/init.d/tomcat6 ] + then + invoke-rc.d tomcat6 stop + fi + if [ -f /etc/init.d/supervisor ] + then + invoke-rc.d supervisor stop + fi + cp /etc/solr/conf/schema.xml /etc/solr/conf/schema.xml.bak + ln -sf /etc/default/databank/schema.xml /etc/solr/conf/schema.xml + ln -sf /var/lib/databank/message_workers/workers_available/worker_broker.conf /etc/supervisor/conf.d/worker_broker.conf + ln -sf /var/lib/databank/message_workers/workers_available/worker_solr.conf /etc/supervisor/conf.d/worker_solr.conf + if [ -f /etc/init.d/tomcat6 ] + then + invoke-rc.d tomcat6 start + fi + if [ -f /etc/init.d/supervisor ] + then + invoke-rc.d supervisor start + fi + else + echo "***********************************************************" + echo "SOLR has not been configured for Databank." + echo "The instructions for doing so are available at" + echo "/usr/share/doc/databank/ConfiguringSOLR.txt" + echo "***********************************************************" + fi +else + echo "***********************************************************" + echo "SOLR has not been configured for Databank." + echo "The instructions for doing so are available at" + echo "/usr/share/doc/databank/ConfiguringSOLR.txt" + echo "***********************************************************" +fi + +#Enable site in apache +a2dissite default +a2dissite default-ssl +a2ensite databank_wsgi + +#Start Apache +invoke-rc.d apache2 reload +invoke-rc.d apache2 start + +db_stop diff --git a/debian/postrm b/debian/postrm new file mode 100644 index 0000000..7bb6e76 --- /dev/null +++ b/debian/postrm @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +# Source debconf library. +. /usr/share/debconf/confmodule + +# Remove my changes to the db. +db_purge diff --git a/debian/preinst b/debian/preinst new file mode 100644 index 0000000..a7d0679 --- /dev/null +++ b/debian/preinst @@ -0,0 +1,10 @@ +#!/bin/bash +set -e + +# stop apache server +# stop django server +if [ -f /etc/init.d/apache2 ] +then + invoke-rc.d apache2 stop +fi + diff --git a/debian/rules b/debian/rules new file mode 100644 index 0000000..917d9bf --- /dev/null +++ b/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + +%: + dh $@ diff --git a/debian/templates b/debian/templates new file mode 100644 index 0000000..017c4c8 --- /dev/null +++ b/debian/templates @@ -0,0 +1,57 @@ +Template: databank/email +Type: string +Description: Adminstrator Email: + The email id will be used to send error messages from Databank. + +Template: databank/smtpserver +Type: string +Default: localhost +Description: SMTP Server: + This is used for sending emails from Databank + +Template: databank/store +Type: string +Default: /silos +Description: Data storage location: + The location where the data will be stored by Databank. + +Template: databank/uri +Type: string +Description: Base URI: + The base uri (domain name) of Databank. + Example: http://example.com/ + +Template: databank/password +Type: string +Description: Administrator password: + The password for user 'admin' + The user 'admin' is the main administartor for Databank and + has access to all parts of Databank + +Template: databank/confsolr +Type: boolean +Description: Can SOLR be configured for Databank? + The search facility in Databank is powered by SOLR. + . + If you choose to configure SOLR now, the existing schema + at /etc/solr/conf/schema.xml will be replaced with Databank's schema + . + If you choose to configure SOLR at a later time, + the instructions to do so are at /usr/share/doc/databank/ConfiguringSOLR.txt + +Template: databank/nosolrconf +Type: note +Description: Configuring SOLR + SOLR has not been configured for Databank. + If you choose to configure SOLR at a later time, + the instructions to do so are at /usr/share/doc/databank/ConfiguringSOLR.txt + +Template: databank/passwdnote +Type: note +Description: Change your password + Databank has been installed successfully and is available at http://localhost + . + The default credntials for the main user is - username: admin password: test + . + On logging in, Please change the default admin password + diff --git a/development.ini b/development.ini index 6bad250..4507c37 100644 --- a/development.ini +++ b/development.ini @@ -1,3 +1,23 @@ +# Copyright (c) 2012 University of Oxford +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # rdfdatabank - Pylons development environment configuration # @@ -8,7 +28,7 @@ debug = true # Uncomment and replace with the address which should receive any error reports #email_to = you@yourdomain.com smtp_server = localhost -error_email_from = paste@localhost +error_email_from = paste@databank [server:main] use = egg:Paste#http @@ -24,33 +44,34 @@ use = egg:rdfdatabank full_stack = true static_files = true -cache_dir = %(here)s/data -beaker.session.key = localdatabank +cache_dir = /var/cache/databank +beaker.session.key = rdfdatabank beaker.session.secret = somesecret -who.config_file = %(here)s/who.ini +who.config_file = /var/lib/databank/who.ini who.log_level = info -#who.log_file = /var/log/databank/who.log who.log_file = stdout -#who.log_file = %(here)s/logs/who.log +#who.log_file = /var/log/databank/who.log redis.host = localhost -granary.store = %(here)s/silos -granary.uri_root = http://192.168.23.133/ +granary.store = /silos +granary.uri_root = http://databank/ -profile.log_filename = %(here)s/logs/profile.log +profile.log_filename = /var/log/databank/profile.log profile.path = /__profile__ -auth.file = %(here)s/passwd -auth.info = %(here)s/rdfdatabank/config/users.py +auth.file = /var/lib/databank/passwd +auth.info = /var/lib/databank/rdfdatabank/config/users.py -doi.config = %(here)s/rdfdatabank/config/doi_config.py -doi.count = %(here)s/rdfdatabank/config/doi_count +doi.config = /var/lib/databank/rdfdatabank/config/doi_config.py +doi.count = /var/lib/databank/rdfdatabank/config/doi_count broadcast.to = redis broadcast.queue = silochanges +metadata.embargoed = False + solr.host = http://localhost:8080/solr naming_rule = [^0-9a-zA-Z_\-\:] formats_served = text/html,text/xhtml,text/plain,application/json,application/rdf+xml,text/xml,text/rdf+n3,application/x-turtle,text/rdf+ntriples,text/rdf+nt @@ -59,6 +80,8 @@ rights = http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4 license = CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode #license = Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. See http://creativecommons.org/licenses/by-nc-sa/3.0/ +api.version = 0.2 + # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings # here: @@ -106,8 +129,6 @@ class = FileHandler level = INFO formatter = generic args = ('/var/log/databank/databank.log', 'w') -#args = ('/opt/RDFDatabank/logs/databank.log', 'w') -#args = ('%(here)s/logs/databank.log', 'w') [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s diff --git a/docs/Databank_VM_Installation.txt b/docs/Databank_VM_Installation.txt index ec62751..a4fc900 100644 --- a/docs/Databank_VM_Installation.txt +++ b/docs/Databank_VM_Installation.txt @@ -2,7 +2,7 @@ Databank VM Setup ------------------------------------------------------------------------------------------------------ I. Virtual machine details ------------------------------------------------------------------------------------------------------ -Ubuntu 10.04.1 32 bit i386 +Ubuntu 11.10 server i386 512MB RAM 8GB Harddrive (all of hard disk not allocated at the time of creation) Network: NAT @@ -23,13 +23,9 @@ Install Grub boot loader to master boot record II. Packages to be Installed ------------------------------------------------------------------------------------------------------ sudo apt-get install build-essential - sudo vim /etc/apt/sources.list - add the following repository to the list - deb http://archive.canonical.com/ lucid partner sudo apt-get update sudo apt-get install openssh-server - sudo apt-get install sun-java6-jdk - Accept the licenses + sudo apt-get install openjdk-6-jre sudo apt-get install python-dev sudo apt-get install curl sudo apt-get install links2 @@ -42,16 +38,18 @@ II. Packages to be Installed sudo apt-get install python-setuptools sudo apt-get install python-libxml2 sudo apt-get install python-libxslt1 + sudo apt-get install python-pylons + sudo apt-get install python-rdflib + sudo apt-get install python-dateutil + sudo easy_install pairtree==0.5.6-T - sudo easy_install pylons==0.9.7 sudo easy_install repoze.who==2.0a4 - sudo easy_install rdflib==2.4.2 + sudo easy_install repoze.who_friendlyform sudo easy_install solrpy - sudo easy_install uuid - sudo easy_install python-dateutil==1.5 Installing profilers in python and pylons to obtain run time performance and other stats - Note: This is only needed in development machines. See the note below about running Pylons in debug mode (section B) + Note: This package is OPTIONAL and is only needed in development machines. + See the note below about running Pylons in debug mode (section B) sudo apt-get install python-profiler sudo easy_install repoze.profile @@ -86,43 +84,57 @@ Accept all of the default options Installing Redis (http://redis.io/) sudo apt-get install redis-server The data directory is at /var/lib/redis and the configuration is at /etc/redis - This installed version 21.2.0-1 + This installed version 2:1.2.0-1 + This is actually version 1.2.0-1 of redis whihc is very old. If on Lucid, download the debian package from Oneiric + sudo apt-get remove --purge redis-server + cd ~ + wget http://ubuntu.intergenia.de/ubuntu//pool/universe/r/redis/redis-server_2.2.11-3_amd64.deb + sudo dpkg -i redis-server_2.2.11-3_amd64.deb Installing the python redis client - wget http://launchpadlibrarian.net/53067907/python-redis_2.0.0-1_all.deb - sudo dpkg -i python-redis_2.0.0-1_all.deb + Get the latest version of the python-redis client from oneiric + wget http://de.archive.ubuntu.com/ubuntu/pool/universe/p/python-redis/python-redis_2.4.5-1_all.deb + sudo dpkg -i python-redis_2.4.5-1_all.deb I initially did sudo apt-get install python-redis. This installed verison 0.6.1, which is the latest version available for Lucid amd64 architecture is 0.6.1, so I removed the package using apt-get remove --purge python-redis - I then downloaded the latest debian package from Natty amd64 (https://launchpad.net/ubuntu/natty/amd64/python-redis/2.0.0-1) - using wget and installed it using dpkg - I did have a look at the backports but from what I saw, the necessary package isn't in there (http://packages.ubuntu.com/lucid-backports/python/) - -Pulling Recordsilo and Databank from github + I then downloaded the latest debian package from Oneiric amd64 using wget and installed it using dpkg + I did have a look at the backports but from what I saw, the necessary package isn't in there + +Installing supervisor - to manage the message workers + sudo apt-get install supervisor + +Installing git + sudo apt-get install git-core git-doc + If you want to clone the repository (to be able to commit changes), + setup git following the instructions in http://help.github.com/linux-set-up-git/ + +Pulling Recordsilo from github Recordsilo is available at https://github.com/anusharanganathan/RecordSilo - RDFDatabank is available at https://github.com/anusharanganathan/RDFDatabank Recordsilo and RDFDatabank and kept in sync so at any period of time the latest egg file in Recordsilo/dist (https://github.com/anusharanganathan/RecordSilo/tree/master/dist) is comaptible with the code in the master branch of RDFDatabank - sudo apt-get install git-core git-doc + Clone/Pull Recordsilo and install the latest vesion. Currently the latest version is 0.4.14. + sudo mkdir ~/git-src cd git-src - If you want to clone the repository (to be able to commit changes), - setup git following the instructions in http://help.github.com/linux-set-up-git/ + git clone git://github.com/anusharanganathan/RecordSilo.git + cd RecordSilo/dist + sudo easy_install RecordSilo-0.4.14-py2.x.egg (2.x - depending on whether you are running python 2.6 / python 2.7) - Clone/Pull Recordsilo and install the latest vesion. Currently the latest version is 0.4.14. - - git clone git@github.com:anusharanganathan/RecordSilo.git - sudo easy_install RecordSilo-0.4.14-py2.6.egg - - Clone/Pull RDFDatabank - cd /opt - sudo mkdir RDFDatabank - sudo chown www-data:demoSystemUser RDFDatabank/ - sudo chmod 775 RDFDatabank/ - git clone git@github.com:anusharanganathan/RDFDatabank.git - - /opt/RDFDatabank is the root directory running the databank instance. You can move this to any other place of your choice. +Pulling Recordsilo and Databank from github + RDFDatabank is available at https://github.com/anusharanganathan/RDFDatabank + + Clone/Pull RDFDatabank into its root directory + Previously Databank was installed in /opt/RDFDatabank/ + Since creating a debian package, Databank is installed in /var/lib/databank/ + Installing it in either place is fine. I shall herewith refer to the location of the Databank root directory with $DB_ROOT + where $DB_ROOT = /var/lib/databank/ (or /opt/RDFDatabank/) + + sudo mkdir -p $DB_ROOT + sudo chown www-data:demoSystemUser $DB_ROOT + sudo chmod 775 $DB_ROOT + git clone git://github.com/dataflow/RDFDatabank.git $DB_ROOT ------------------------------------------------------------------------------------------------------ III. Customizing Databank to your environment @@ -157,19 +169,30 @@ In the file production.ini, edit the folllowing: ------------------------------------------------------------------------------------------------------ IV. Create the main admin user to access Databank + +TODO: This section uses htpasswd and a python file to manage user groups and roles. + This would best be replaced by a DB lookup or similar. ------------------------------------------------------------------------------------------------------ -Create a file 'passwd' in the root directory of RDFDatabank using 'htpasswd' and add a user to the passwd file. -OR copy the file passwd-default to passwd +1. Create a file 'passwd' in the root directory of RDFDatabank using 'htpasswd' and add a user to the passwd file. + OR copy the file passwd-default to passwd + + The passwd-default file has users whose username, password and roles are as given in the file rdfdatabank/tests/userRolesForTesting.txt. + These user accounts are just for demonstration and would have to be changed for a production instance. -To create the pssswd file - cd RDFDatabank/ - htpasswd -c passwd admin + To create a new pssswd file with the username 'admin'. + Note: To start off you just need to create this one username and password. This user will be the main databank administrator. + cd RDFDatabank/ + htpasswd -c passwd admin - [enter admin password] + [enter admin password] -Create and add data to the file rdfdatabank/config/users.py. - You would need to create a file called 'users.py' in add data for the main admin user (added earlier into htpasswd) by hand currently. -OR copy the file users-default.py to rdfdatabank/config/users.py +2. Create and add data to the file rdfdatabank/config/users.py. + OR copy the file rdfdatabank/config/users-default.py to rdfdatabank/config/users-.py. + + Currently, you need to create a file called 'users.py' in rdfdatabank/config/ and populate the file with the data for each of the users you had added into the passwd file. + You just need to define the user for the main admin account user ('admin' - added earlier into htpasswd). You can create further users using the Databank web interface. + + Alternatively, you can copy the file rdfdatabank/config/users-default.py to rdfdatabank/config/users.py If creating the users.py file: The data to add should look like this @@ -192,9 +215,7 @@ _USERS = { Examples for owner: 'owner': '*', - 'owner': 'my_first_silo,sandbox,my_second_silo', - - The file has a variable _USERS used by ./rdfdatabank/lib/ident_md.py which would best be replaced by a DB lookup or similar. + 'owner': 'my_first_silo,sandbox,my_second_silo', ------------------------------------------------------------------------------------------------------ V. Integrate Databank with Datacite, for minting DOIs (this section is optional) @@ -236,20 +257,20 @@ class OxDataciteDoi(): VI. Integrate Pylons with Apache ------------------------------------------------------------------------------------------------------ 1. Edit the file dispatch.wsgi - Note: you only need to do this step if the root directory running the databank instance is not /opt/RDFDatabank) + Note: you only need to do this step if the root directory running the databank instance is not /var/lib/databank) In the mod_wsgi/dispatch.wsgi file, edit the location of your databank instance. - Replace the string '/opt/RDFDatabank' with the complete path to the directory where you have installed Databank. + Replace the string '/var/lib/databank' with the complete path to the directory where you have installed Databank ($DB_ROOT). 2. Add a new site in apache sites-available called databank-wsgi - There is a file provided to you in the docs dir called databank_wsgi (/opt/RDFDatabank/docs/apache_config/databank_wsgi). - If the root directory running the databank instance is not /opt/RDFDatabank, edit the location of your databank instance, - replacing the string '/opt/RDFDatabank' with the complete path to the directory where you have installed Databank. + There is a file provided to you in the docs dir called databank_wsgi ($DB_ROOT/docs/apache_config/databank_wsgi). + If the root directory running the databank instance is not /var/lib/databank, edit the location of your databank instance, + replacing the string '/var/lib/databank' with the complete path to the directory where you have installed Databank ($DB_ROOT). The apache log files are configured to be stored in /var/log/apache2/. Agaian you can change this location. Copy the file docs/apache_config/databank_wsgi to /etc/apache/sites-available. - $sudo cp /opt/RDFDatabank/docs/apache_config/databank_wsgi /etc/apache2/sites-available/ + $sudo cp $DB_ROOT/docs/apache_config/databank_wsgi /etc/apache2/sites-available/ 3. Enable mod_wsgi and the site databank_wsgi. If the site default is enabled, disable that site. a2dissite 000-default @@ -261,20 +282,58 @@ VII. Setup SOLR ------------------------------------------------------------------------------------------------------ Copy the solr schema file from docs into your solr conf sudo /etc/init.d/tomcat6 stop - cp /opt/RDFDatabank/docs/solr_config/conf/schema.xml /etc/solr/conf/ + sudo cp $DB_ROOT/docs/solr_config/conf/schema.xml /etc/solr/conf/ sudo /etc/init.d/tomcat6 start ------------------------------------------------------------------------------------------------------ -VIII. Making sure all of the needed folders are available and apache has access to all the needed parts +VIII. Setup Supervisor +------------------------------------------------------------------------------------------------------ +Items are indexed in SOLR from Databank, through redis using message queues +The workers that run on these message queues are managed using supervisor + + The related files are available in $DB_ROOT/message_workers + + Modify the file loglines.cfg - if the solr end point is not http://localhost:8080/solr or databank root is not /var/lib/databank + [worker_solr] + solrurl = http://localhost:8080/solr + dbroot = /var/lib/databank + + Modify the file workers_available/worker_broker.conf - if databank root is not /var/lib/databank + [program:worker_broker] + directory = /var/lib/databank/message_workers/ + command = /var/lib/databank/message_workers/broker.py %(process_num)s + stdout_logfile = /var/log/databank/broker.log + + Modify the file workers_available/worker_solr.conf - if databank root is not /var/lib/databank + [program:worker_solr] + directory = /var/lib/databank/message_workers/ + command = /var/lib/databank/message_workers/solr_worker.py %(process_num)s + stdout_logfile = /var/log/databank/solr_worker.log + + Stop supervisor, copy the following files to /etc/supervisor/conf.d/ and start supervisor + $ sudo /etc/init.d/supervisor stop + $ sudo cp workers_available/worker_broker.conf /etc/supervisor/conf.d/ + $ sudo cp workers_available/worker_solr.conf /etc/supervisor/conf.d + $ sudo /etc/init.d/supervisor start + + The controller for supervisor can be invoked with the command 'supervisorctl' + sudo supervisorctl + + This will list all of the jobs manged by supervisor and their current status. + You can start / stop / restart jobs from withon the controller. + For more info on supervisor, read http://supervisord.org/index.html + +------------------------------------------------------------------------------------------------------ +IX. Making sure all of the needed folders are available and apache has access to all the needed parts ------------------------------------------------------------------------------------------------------ Apache runs as user www-data. Make sure the user www-data is able to read write to the following locations - /opt/RDFDatabank + $DB_ROOT /silos (create if not available) /var/log/databank (create if not available) Change permission, so www-data has access to RDFDatabank - sudo chown -R www-data:demoSystemUser /opt/RDFDatabank/ - sudo chmod -R 775 /opt/RDFDatabank/ + sudo chown -R www-data:demoSystemUser $DB_ROOT + sudo chmod -R 775 $DB_ROOT Create a folder called /silos and change it's owner and permission. This is where the silos are going to be stored sudo mkdir /silos @@ -287,7 +346,7 @@ Create a folder called databank in /var/log/databank and change it's owner and p sudo chmod -R 775 /var/log/databank ------------------------------------------------------------------------------------------------------ -IX. Test your Pylons installation +X. Test your Pylons installation ------------------------------------------------------------------------------------------------------ Restart apache sudo /etc/init.d/apache2 restart @@ -300,15 +359,15 @@ If you have problems with authorization, you can change the who.log_level to deb and restart apache so you can check the logs to see what's happening. The who.log_file is available at /var/log/databank/who.log ------------------------------------------------------------------------------------------------------ -X. Run the test code and make sure all the tests pass +XI. Run the test code and make sure all the tests pass ------------------------------------------------------------------------------------------------------ -The test code is located at /opt/RDFDatabank/rdfdatabank/tests +The test code is located at $DB_ROOT/rdfdatabank/tests There are two test files - TestSubmission.py and TestSubmission_submitter.py TestSubmission.py test all of the different functionality TestSubmission_submitter.py tests the different user roles and permissions -Both of these tests use the configuration file RDFDatabankConfig.py, which you may need to mkodify +Both of these tests use the configuration file RDFDatabankConfig.py, which you may need to modify granary_uri_root="http://databank" This needs to be the same value as granary.uri_root in the production.ini file (or development.ini file if usign that instead) endpointhost="localhost" @@ -322,7 +381,7 @@ Both of these tests use the configuration file RDFDatabankConfig.py, which you m To run the tests Make sure databank is running (see section IX) - cd /opt/RDFDatabank/rdfdatabank/tests + cd $DB_ROOT/rdfdatabank/tests python TestSubmission.py (to run the tests in TestSubmission.py) python TestSubmission_submitter.py (to run the tests in TestSubmission_submitter.py) @@ -334,7 +393,7 @@ If you would like to run Pylons in debug mode from the command line and dump all Open the file development.ini and make the changes as in section III (except may be who.log_file). Note: development.ini is similar to and essentially a copy of production.ini, except in the configuration of logs and debug levels. - To run pylons in debug mode, modify set 'debug' to 'true' in the '[DEFAULT]' section + To run pylons in debug mode, set 'debug' to 'true' in the '[DEFAULT]' section [DEFAULT] debug = true If you run pylons in debug mode, the error stack will be displayed on the browser with the state variables and their values. diff --git a/docs/Dependencies.pdf b/docs/Dependencies.pdf new file mode 100644 index 0000000..e8cabba Binary files /dev/null and b/docs/Dependencies.pdf differ diff --git a/docs/accessLogWithheaderInfo_2011_03_16.csv b/docs/accessLogWithheaderInfo_2011_03_16.csv deleted file mode 100644 index aea9df5..0000000 Binary files a/docs/accessLogWithheaderInfo_2011_03_16.csv and /dev/null differ diff --git a/docs/accessLogWithheaderInfo_2011_03_16.pdf b/docs/accessLogWithheaderInfo_2011_03_16.pdf deleted file mode 100644 index 962790f..0000000 Binary files a/docs/accessLogWithheaderInfo_2011_03_16.pdf and /dev/null differ diff --git a/docs/apache_config/databank_development_wsgi b/docs/apache_config/databank_development_wsgi index 66a6cdc..b196759 100644 --- a/docs/apache_config/databank_development_wsgi +++ b/docs/apache_config/databank_development_wsgi @@ -1,5 +1,5 @@ - ServerName databank.bodleian.ox.ac.uk + #ServerName databank.com # Logfiles ErrorLog /var/log/apache2/databank-error.log @@ -10,9 +10,9 @@ WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On # Setup mod_wsgi - WSGIScriptAlias / /opt/RDFDatabank/mod_wsgi/dispatch_development.wsgi + WSGIScriptAlias / /var/lib/databank/mod_wsgi/dispatch_development.wsgi - + Order deny,allow Allow from all diff --git a/docs/apache_config/databank_wsgi b/docs/apache_config/databank_wsgi index 89bc9f0..071d498 100644 --- a/docs/apache_config/databank_wsgi +++ b/docs/apache_config/databank_wsgi @@ -1,5 +1,5 @@ - ServerName databank.bodleian.ox.ac.uk + #ServerName databank.com # Logfiles ErrorLog /var/log/apache2/databank-error.log @@ -10,9 +10,9 @@ WSGIApplicationGroup %{GLOBAL} WSGIPassAuthorization On # Setup mod_wsgi - WSGIScriptAlias / /opt/RDFDatabank/mod_wsgi/dispatch.wsgi + WSGIScriptAlias / /var/lib/databank/mod_wsgi/dispatch.wsgi - + Order deny,allow Allow from all diff --git a/docs/assigning_dois/DOI Syntax for datasets in DataBank.docx b/docs/assigning_dois/DOI Syntax for datasets in DataBank.docx deleted file mode 100755 index 25db4a3..0000000 Binary files a/docs/assigning_dois/DOI Syntax for datasets in DataBank.docx and /dev/null differ diff --git a/docs/assigning_dois/DOISyntax.doc b/docs/assigning_dois/DOISyntax.doc new file mode 100644 index 0000000..f9c7866 Binary files /dev/null and b/docs/assigning_dois/DOISyntax.doc differ diff --git a/docs/assigning_dois/ImplementingDOIs.ppt b/docs/assigning_dois/ImplementingDOIs.ppt index ea2d7cb..6b41109 100755 Binary files a/docs/assigning_dois/ImplementingDOIs.ppt and b/docs/assigning_dois/ImplementingDOIs.ppt differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_pyhonTestMessage.txt deleted file mode 100644 index 49b3f07..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_pyhonTestMessage.txt +++ /dev/null @@ -1,91 +0,0 @@ - -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already impo rted from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.py, but /usr/local/lib/python2.6/dist- packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -.............FFF..FFEE........ -====================================================================== -ERROR: Delete embargo information - POST embargo_change to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1394, in testDeleteEmbargo - expect_status=200, expect_reason="OK", expect_type="application/json") - File "../TestLib/SparqlQueryTestCase.py", line 182, in doHTTP_GET - if (expect_type.lower() == "application/json"): responsedata = simplejson.loads(responsedata) - File "/usr/lib/pymodules/python2.6/simplejson/__init__.py", line 307, in loads - return _default_decoder.decode(s) - File "/usr/lib/pymodules/python2.6/simplejson/decoder.py", line 338, in decode - raise ValueError(errmsg("Extra data", s, end, len(s))) -ValueError: Extra data: line 1 column 4 - line 11 column 1372 (char 4 - 1664) - -====================================================================== -ERROR: Modify embargo information - POST embargo_change, embargo, embargo_until to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1457, in testChangeEmbargo - expect_status=200, expect_reason="OK", expect_type="application/json") - File "../TestLib/SparqlQueryTestCase.py", line 182, in doHTTP_GET - if (expect_type.lower() == "application/json"): responsedata = simplejson.loads(responsedata) - File "/usr/lib/pymodules/python2.6/simplejson/__init__.py", line 307, in loads - return _default_decoder.decode(s) - File "/usr/lib/pymodules/python2.6/simplejson/decoder.py", line 338, in decode - raise ValueError(errmsg("Extra data", s, end, len(s))) -ValueError: Extra data: line 1 column 4 - line 11 column 1394 (char 4 - 1686) - -====================================================================== -FAIL: Update file in dataset - POST file to /silo_name/datasets/dataset_name (x 2) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 592, in testFileUpdate - self.assertEqual(zipdata, zipfile, "Difference between local and remote zipfile!") -AssertionError: Difference between local and remote zipfile! - -====================================================================== -FAIL: Upload files to a dataset - POST file to /silo_name/datasets/dataset_name. Access each of the versions and the files in that version ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 755, in testGetDatasetByVersion - expect_status=404, expect_reason="Not Found") - File "../TestLib/SparqlQueryTestCase.py", line 180, in doHTTP_GET - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 404 - -====================================================================== -FAIL: POST manifest to dataset - POST manifest.rdf to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1006, in testMetadataFileUpdate - zipdata = self.updateTestSubmissionZipfile(file_to_upload="manifest2.rdf", filename="manifest.rdf") - File "TestSubmission.py", line 111, in updateTestSubmissionZipfile - expect_status=204, expect_reason="No Content") - File "../TestLib/SparqlQueryTestCase.py", line 207, in doHTTP_POST - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 204 - -====================================================================== -FAIL: PUT file contents to existing filename - PUT file contents to /silo_name/datasets/dataset_name/file_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1212, in testPutUpdateFile - self.assertEqual(zipdata3, zipfile, "Difference between local and remote zipfile!") -AssertionError: Difference between local and remote zipfile! - -====================================================================== -FAIL: Add metadata to manifest - PUT metadata to /silo_name/datasets/dataset_name/manifest.rdf ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1294, in testPutMetadataFile - expect_status=204, expect_reason="No Content", expect_type="text/plain") - File "../TestLib/SparqlQueryTestCase.py", line 237, in doHTTP_PUT - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 204 - ----------------------------------------------------------------------- -Ran 30 tests in 91.021s - -FAILED (failures=5, errors=2) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_wiresharkMessage.pcap deleted file mode 100644 index 4e7e68d..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run1_5Fail_2Error_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_pyhonTestMessage.txt deleted file mode 100644 index 4295489..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_pyhonTestMessage.txt +++ /dev/null @@ -1,39 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testDeleteEmbargo -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Delete embargo information - POST embargo_change to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 204 Updated -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ERROR - -====================================================================== -ERROR: Delete embargo information - POST embargo_change to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1394, in testDeleteEmbargo - expect_status=200, expect_reason="OK", expect_type="application/json") - File "../TestLib/SparqlQueryTestCase.py", line 182, in doHTTP_GET - if (expect_type.lower() == "application/json"): responsedata = simplejson.loads(responsedata) - File "/usr/lib/pymodules/python2.6/simplejson/__init__.py", line 307, in loads - return _default_decoder.decode(s) - File "/usr/lib/pymodules/python2.6/simplejson/decoder.py", line 338, in decode - raise ValueError(errmsg("Extra data", s, end, len(s))) -ValueError: Extra data: line 1 column 4 - line 11 column 1372 (char 4 - 1664) - ----------------------------------------------------------------------- -Ran 1 test in 19.271s - -FAILED (errors=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap deleted file mode 100644 index 4402859..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_pyhonTestMessage.txt deleted file mode 100644 index 04b281c..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_pyhonTestMessage.txt +++ /dev/null @@ -1,39 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testChangeEmbargo -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Modify embargo information - POST embargo_change, embargo, embargo_until to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 204 Updated -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ERROR - -====================================================================== -ERROR: Modify embargo information - POST embargo_change, embargo, embargo_until to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1457, in testChangeEmbargo - expect_status=200, expect_reason="OK", expect_type="application/json") - File "../TestLib/SparqlQueryTestCase.py", line 182, in doHTTP_GET - if (expect_type.lower() == "application/json"): responsedata = simplejson.loads(responsedata) - File "/usr/lib/pymodules/python2.6/simplejson/__init__.py", line 307, in loads - return _default_decoder.decode(s) - File "/usr/lib/pymodules/python2.6/simplejson/decoder.py", line 338, in decode - raise ValueError(errmsg("Extra data", s, end, len(s))) -ValueError: Extra data: line 1 column 4 - line 11 column 1391 (char 4 - 1683) - ----------------------------------------------------------------------- -Ran 1 test in 24.111s - -FAILED (errors=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap deleted file mode 100644 index ca89a28..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_pyhonTestMessage.txt deleted file mode 100644 index 8911a45..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_pyhonTestMessage.txt +++ /dev/null @@ -1,37 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testFileUpdate -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Update file in dataset - POST file to /silo_name/datasets/dataset_name (x 2) ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -FAIL - -====================================================================== -FAIL: Update file in dataset - POST file to /silo_name/datasets/dataset_name (x 2) ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 592, in testFileUpdate - self.assertEqual(zipdata, zipfile, "Difference between local and remote zipfile!") -AssertionError: Difference between local and remote zipfile! - ----------------------------------------------------------------------- -Ran 1 test in 22.510s - -FAILED (failures=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap deleted file mode 100644 index 2196a96..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_pyhonTestMessage.txt deleted file mode 100644 index b8c9c1c..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_pyhonTestMessage.txt +++ /dev/null @@ -1,69 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testGetDatasetByVersion -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Upload files to a dataset - POST file to /silo_name/datasets/dataset_name. Access each of the versions and the files in that version ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version0 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -FAIL - -====================================================================== -FAIL: Upload files to a dataset - POST file to /silo_name/datasets/dataset_name. Access each of the versions and the files in that version ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 755, in testGetDatasetByVersion - expect_status=404, expect_reason="Not Found") - File "../TestLib/SparqlQueryTestCase.py", line 180, in doHTTP_GET - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 404 - ----------------------------------------------------------------------- -Ran 1 test in 16.518s - -FAILED (failures=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap deleted file mode 100644 index 77c064f..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_pyhonTestMessage.txt deleted file mode 100644 index 3b5d7d6..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_pyhonTestMessage.txt +++ /dev/null @@ -1,37 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testMetadataFileUpdate -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -POST manifest to dataset - POST manifest.rdf to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 200 OK -FAIL - -====================================================================== -FAIL: POST manifest to dataset - POST manifest.rdf to /silo_name/datasets/dataset_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1006, in testMetadataFileUpdate - zipdata = self.updateTestSubmissionZipfile(file_to_upload="manifest2.rdf", filename="manifest.rdf") - File "TestSubmission.py", line 111, in updateTestSubmissionZipfile - expect_status=204, expect_reason="No Content") - File "../TestLib/SparqlQueryTestCase.py", line 207, in doHTTP_POST - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 204 - ----------------------------------------------------------------------- -Ran 1 test in 0.437s - -FAILED (failures=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap deleted file mode 100644 index 9c8178e..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_pyhonTestMessage.txt deleted file mode 100644 index 348d072..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_pyhonTestMessage.txt +++ /dev/null @@ -1,43 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testPutUpdateFile -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -PUT file contents to existing filename - PUT file contents to /silo_name/datasets/dataset_name/file_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testrdf3.zip -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testrdf3.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -FAIL - -====================================================================== -FAIL: PUT file contents to existing filename - PUT file contents to /silo_name/datasets/dataset_name/file_name ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1212, in testPutUpdateFile - self.assertEqual(zipdata3, zipfile, "Difference between local and remote zipfile!") -AssertionError: Difference between local and remote zipfile! - ----------------------------------------------------------------------- -Ran 1 test in 22.677s - -FAILED (failures=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap deleted file mode 100644 index 3d2866c..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_pyhonTestMessage.txt deleted file mode 100644 index 5028fe5..0000000 --- a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_pyhonTestMessage.txt +++ /dev/null @@ -1,35 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testPutMetadataFile -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Add metadata to manifest - PUT metadata to /silo_name/datasets/dataset_name/manifest.rdf ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/manifest.rdf -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/manifest.rdf -DEBUG:SparqlQueryTestCase:Status: 200 OK -FAIL - -====================================================================== -FAIL: Add metadata to manifest - PUT metadata to /silo_name/datasets/dataset_name/manifest.rdf ----------------------------------------------------------------------- -Traceback (most recent call last): - File "TestSubmission.py", line 1294, in testPutMetadataFile - expect_status=204, expect_reason="No Content", expect_type="text/plain") - File "../TestLib/SparqlQueryTestCase.py", line 237, in doHTTP_PUT - expect_status=expect_status, expect_reason=expect_reason) - File "../TestLib/SparqlQueryTestCase.py", line 164, in doRequest - if expect_status != "*": self.assertEqual(response.status, expect_status) -AssertionError: 200 != 204 - ----------------------------------------------------------------------- -Ran 1 test in 3.759s - -FAILED (failures=1) diff --git a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap deleted file mode 100644 index 6c936a6..0000000 Binary files a/docs/databank_test_run_for_204_response/test1-204_ppdated_returns_content-5_fail_2_error_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_pyhonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_pyhonTestMessage.txt deleted file mode 100644 index 244de1a..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_pyhonTestMessage.txt +++ /dev/null @@ -1,8 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -.............................. ----------------------------------------------------------------------- -Ran 30 tests in 21.088s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_wiresharkMessage.pcap deleted file mode 100644 index f8f2aec..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run1_all_pass_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_pythonTestMessage.txt deleted file mode 100644 index 3c7429f..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_pythonTestMessage.txt +++ /dev/null @@ -1,25 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testDeleteEmbargo -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Delete embargo information - POST embargo_change to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 204 Updated -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.369s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap deleted file mode 100644 index ff1f1d8..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run2_testDeleteEmbargo_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_pythonTestMessage.txt deleted file mode 100644 index 8bfb78b..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_pythonTestMessage.txt +++ /dev/null @@ -1,25 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testChangeEmbargo -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Modify embargo information - POST embargo_change, embargo, embargo_until to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 204 Updated -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.431s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap deleted file mode 100644 index 88ae69e..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run3_testChangeEmbargo_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_pythonTestMessage.txt deleted file mode 100644 index 44cacca..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_pythonTestMessage.txt +++ /dev/null @@ -1,31 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testFileUpdate -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Update file in dataset - POST file to /silo_name/datasets/dataset_name (x 2) ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.481s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap deleted file mode 100644 index dbe2151..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run4_testFileUpdate_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_pythonTestMessage.txt deleted file mode 100644 index c9c0879..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_pythonTestMessage.txt +++ /dev/null @@ -1,99 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testGetDatasetByVersion -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Upload files to a dataset - POST file to /silo_name/datasets/dataset_name. Access each of the versions and the files in that version ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version0 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version0 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission/version0 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version1 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip/version1 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission/version1 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version2 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip/version2 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip/version2 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission/version2 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version3 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip/version3 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip/version3 -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission/version3 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version4 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip/version4 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip/version4 -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission/version4 -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/version5 -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir2.zip/version5 -DEBUG:SparqlQueryTestCase:Status: 404 Not Found -ok - ----------------------------------------------------------------------- -Ran 1 test in 1.915s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap deleted file mode 100644 index f9dabb0..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run5_testGetDatasetByVersion_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_pythonTestMessage.txt deleted file mode 100644 index 44d3546..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_pythonTestMessage.txt +++ /dev/null @@ -1,27 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testMetadataFileUpdate -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -POST manifest to dataset - POST manifest.rdf to /silo_name/datasets/dataset_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.490s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap deleted file mode 100644 index 3e18a68..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run6_testMetadataFileUpdate_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_pythonTestMessage.txt deleted file mode 100644 index 3cbc0d9..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_pythonTestMessage.txt +++ /dev/null @@ -1,41 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testPutUpdateFile -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -PUT file contents to existing filename - PUT file contents to /silo_name/datasets/dataset_name/file_name ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testrdf3.zip -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testrdf3.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testrdf3.zip -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/testdir.zip/version1 -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.819s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap deleted file mode 100644 index 3bc3d31..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run7_testPutUpdateFile_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_pythonTestMessage.txt b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_pythonTestMessage.txt deleted file mode 100644 index 7636f96..0000000 --- a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_pythonTestMessage.txt +++ /dev/null @@ -1,27 +0,0 @@ -databankadmin@databank:/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank$ python TestSubmission.py testPutMetadataFile -/opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/plugin.py:94: UserWarning: Module rdflib was already imported from /opt/RDFDatabank/rdfdatabank/tests/RDFDatabank/rdflib/__init__.pyc, but /usr/local/lib/python2.6/dist-packages/rdflib-2.4.2-py2.6-linux-i686.egg is being added to sys.path - from pkg_resources import iter_entry_points -Add metadata to manifest - PUT metadata to /silo_name/datasets/dataset_name/manifest.rdf ... DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointhost databank.ora.ox.ac.uk: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpath /sandbox/: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointuser admin: -DEBUG:SparqlQueryTestCase:setRequestEndPoint: endpointpass test: -DEBUG:SparqlQueryTestCase:DELETE http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:POST http://databank.ora.ox.ac.uk/sandbox/datasets/ -DEBUG:SparqlQueryTestCase:Status: 201 Created -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/manifest.rdf -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:PUT http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission/manifest.rdf -DEBUG:SparqlQueryTestCase:Status: 204 No Content -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/datasets/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -DEBUG:SparqlQueryTestCase:GET http://databank.ora.ox.ac.uk/sandbox/states/TestSubmission -DEBUG:SparqlQueryTestCase:Status: 200 OK -ok - ----------------------------------------------------------------------- -Ran 1 test in 0.538s - -OK diff --git a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap b/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap deleted file mode 100644 index f69a215..0000000 Binary files a/docs/databank_test_run_for_204_response/test2-204_updated_returns_no_content-all_pass_in_test_run/run8_testPutMetadataFile_wiresharkMessage.pcap and /dev/null differ diff --git a/docs/digitalbooksMetadataWrite.py b/docs/digitalbooksMetadataWrite.py deleted file mode 100644 index 365ed8d..0000000 --- a/docs/digitalbooksMetadataWrite.py +++ /dev/null @@ -1,50 +0,0 @@ -from solr import SolrConnection -import json -import codecs - -solrhost = "http://localhost:8080/solr" -s = SolrConnection(solrhost) - -fieldnames = ['silo', 'id', 'uuid', 'aggregatedResource', 'created', 'creator', 'currentVersion', 'date', 'dateAccepted', 'dateCopyrighted', 'dateSubmitted', 'description', 'embargoStatus', 'embargoedUntilDate', 'mediator', 'isPartOf', 'isVersionOf', 'license', 'modified', 'publisher', 'rights', 'subject', 'timestamp', 'title', 'type'] - -solr_params = {} -solr_params['q'] = "silo:digitalbooks" -solr_params['wt'] = 'json' -solr_params['fl'] = ','.join(fieldnames) -solr_params['rows'] = 500000 -solr_params['start'] = 0 - -solr_response = s.raw_query(**solr_params) - -numFound = 0 -docs = None -fname = "digitalbooks.csv" -delimiter = '$' - -if solr_response: - ans = json.loads(solr_response) - numFound = ans['response'].get('numFound',None) - try: - numFound = int(numFound) - except: - numFound = 0 - docs = ans['response'].get('docs',None) - if numfound > 0 and docs: - out_f = codecs.open(fname, 'a', 'utf-8') - for row in docs: - row_val = [] - for name in fieldnames: - if name in row and row[name] and isinstance(row[name], basestring): - row_val.append(row[name]) - elif name in row and row[name] and isinstance(row[name], list): - row_val.append(";".join(row[name])) - else: - row_val.append("") - if row_val: - out_f.write("%s\n" %delimiter.join(row_val) - out_f.close() - else: - print 'The search resulted in no documents' -else: - print 'The search resulted in no matches' - diff --git a/docs/index.txt b/docs/index.txt deleted file mode 100644 index b3a9628..0000000 --- a/docs/index.txt +++ /dev/null @@ -1,19 +0,0 @@ -rdfdatabank -+++++++++++ - -This is the main index page of your documentation. It should be written in -`reStructuredText format `_. - -You can generate your documentation in HTML format by running this command:: - - setup.py pudge - -For this to work you will need to download and install `buildutils`_, -`pudge`_, and `pygments`_. The ``pudge`` command is disabled by -default; to ativate it in your project, run:: - - setup.py addcommand -p buildutils.pudge_command - -.. _buildutils: http://pypi.python.org/pypi/buildutils -.. _pudge: http://pudge.lesscode.org/ -.. _pygments: http://pygments.org/ diff --git a/docs/indexingSolrRecordsUsingSupervisord/LogConfigParser.py b/docs/indexingSolrRecordsUsingSupervisord/LogConfigParser.py deleted file mode 100644 index ef9aec1..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/LogConfigParser.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python - -import ConfigParser, os - -class Config(ConfigParser.ConfigParser): - DEFAULT_CONFIG_FILE = "loglines.cfg" - def __init__(self, config_file=DEFAULT_CONFIG_FILE): - ConfigParser.ConfigParser.__init__(self) - if os.path.exists(config_file) and os.path.isfile(config_file): - self.read(config_file) - self.validate() - - def validate(self): - pass diff --git a/docs/indexingSolrRecordsUsingSupervisord/getObjects.py b/docs/indexingSolrRecordsUsingSupervisord/getObjects.py deleted file mode 100644 index 9836564..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/getObjects.py +++ /dev/null @@ -1,86 +0,0 @@ -import os -from rdfdatabank.lib.broadcast import BroadcastToRedis -from pylons import config - -def get_objs_in_dir(items_list, dirname, fnames): - for fname in fnames: - a = os.path.join(dirname,fname) - if fname == 'obj': - item = a.split('pairtree_root')[1].strip('/').split('obj')[0].replace('/', '') - silo = a.split('pairtree_root')[0].strip('/').split('/')[-1] - if not (silo, item) in items_list: - items_list.append((silo, item)) - return - -def broadcast_links(src_dir): - links_list = [] - os.path.walk(src_dir,get_objs_in_dir,links_list) - b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) - - for silo, item in links_list: - b.creation(silo, item) - return - -src_dirs = [ -'/silos/admiral/pairtree_root', -'/silos/digitaltest/pairtree_root', -'/silos/eidcsr/pairtree_root', -'/silos/general/pairtree_root', -'/silos/ww1archives/pairtree_root', -'/silos/digitalbooks/pairtree_root/30', -'/silos/digitalbooks/pairtree_root/og/-4/00', -'/silos/digitalbooks/pairtree_root/og/-4/01', -'/silos/digitalbooks/pairtree_root/og/-3/00', -'/silos/digitalbooks/pairtree_root/og/-3/01', -'/silos/digitalbooks/pairtree_root/og/-3/02', -'/silos/digitalbooks/pairtree_root/og/-3/03', -'/silos/digitalbooks/pairtree_root/og/-3/04', -'/silos/digitalbooks/pairtree_root/og/-3/05', -'/silos/digitalbooks/pairtree_root/og/-3/06', -'/silos/digitalbooks/pairtree_root/og/-3/15', -'/silos/digitalbooks/pairtree_root/og/-3/16', -'/silos/digitalbooks/pairtree_root/og/-3/18', -'/silos/digitalbooks/pairtree_root/og/-3/20', -'/silos/digitalbooks/pairtree_root/og/-3/61', -'/silos/digitalbooks/pairtree_root/og/-3/90', -'/silos/digitalbooks/pairtree_root/og/-3/93', -'/silos/digitalbooks/pairtree_root/og/-5/00', -'/silos/digitalbooks/pairtree_root/og/-5/01', -'/silos/digitalbooks/pairtree_root/og/-5/02', -'/silos/digitalbooks/pairtree_root/og/-5/03', -'/silos/digitalbooks/pairtree_root/og/-5/04', -'/silos/digitalbooks/pairtree_root/og/-5/09', -'/silos/digitalbooks/pairtree_root/og/-5/31', -'/silos/digitalbooks/pairtree_root/og/-5/32', -'/silos/digitalbooks/pairtree_root/og/-5/33', -'/silos/digitalbooks/pairtree_root/og/-5/50', -'/silos/digitalbooks/pairtree_root/og/-5/55', -'/silos/digitalbooks/pairtree_root/og/-5/56', -'/silos/digitalbooks/pairtree_root/og/-5/90', -'/silos/digitalbooks/pairtree_root/og/-5/91', -'/silos/digitalbooks/pairtree_root/og/-5/96', -'/silos/digitalbooks/pairtree_root/og/-5/97', -'/silos/digitalbooks/pairtree_root/og/-6/00', -'/silos/digitalbooks/pairtree_root/og/-6/50', -'/silos/digitalbooks/pairtree_root/og/-6/81', -'/silos/digitalbooks/pairtree_root/og/-6/90', -'/silos/digitalbooks/pairtree_root/og/-N/08', -'/silos/digitalbooks/pairtree_root/og/-N/10', -'/silos/digitalbooks/pairtree_root/og/-N/11', -'/silos/digitalbooks/pairtree_root/og/-N/12', -'/silos/digitalbooks/pairtree_root/og/-N/13', -'/silos/digitalbooks/pairtree_root/og/-N/14', -'/silos/digitalbooks/pairtree_root/og/-N/15', -'/silos/digitalbooks/pairtree_root/og/-N/16', -'/silos/digitalbooks/pairtree_root/og/-N/17', -'/silos/digitalbooks/pairtree_root/og/-N/32', -'/silos/digitalbooks/pairtree_root/og/-N/50' -] - -for src_dir in src_dirs: - print "starting", src_dir - links_list = [] - os.path.walk(src_dir,get_objs_in_dir,links_list) - b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) - for silo, item in links_list: - b.creation(silo, item) \ No newline at end of file diff --git a/docs/indexingSolrRecordsUsingSupervisord/loglines.cfg b/docs/indexingSolrRecordsUsingSupervisord/loglines.cfg deleted file mode 100644 index 7c2812a..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/loglines.cfg +++ /dev/null @@ -1,55 +0,0 @@ -# CONFIGURE THE FOLLOWING: -[supervisor] -port = 127.0.0.1:9001 -username = guest -password = guest - -# Configure the following to let supervisor control the redis instance -# or comment it out otherwise. -[process_redis] -# add the correct path to the redis instance here -command = redis-1.2.6/redis-server - -# Configure the following to let supervisor control the redis instance -# or comment it out otherwise. -[process_solr] -# add the correct path to the redis instance here -command = /opt/solr/start_solr.sh - -[worker_solr] -listento = solrindex -command = ./solr_worker.py -# Time in seconds to sleep if there is nothing on the queue -solrurl = http://localhost:8983/solr -idletime = 1 -stdout_logfile = workerlogs/solr_worker.log -numprocs = 2 - -[worker_broker] -listento = silochanges -command = ./broker.py -fanout = auditlog, solrindex -fanout_status_queue = broker_temp -# Time in seconds to sleep if there is nothing on the queue -idletime = 1 -stdout_logfile = workerlogs/broker.log -numprocs = 2 - -[logger_auditlogger] -listento = auditlog -command = ./logfromqueue.py -logfile = logs/audit.log -stdout_logfile = workerlogs/auditlogger.log - -# DEFAULT VALUES FOLLOW -############################## -[redis] -host = localhost -port = 6379 -db = 0 - -[redis_test] -host = localhost -port = 6379 -db = 1 - diff --git a/docs/indexingSolrRecordsUsingSupervisord/solrFields.py b/docs/indexingSolrRecordsUsingSupervisord/solrFields.py deleted file mode 100644 index 555a3bc..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/solrFields.py +++ /dev/null @@ -1,102 +0,0 @@ -solr_fields_mapping = { - "silo":"silo", - "id":"id", - "uuid":"uuid", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"type", - "http://vocab.ox.ac.uk/dataset/schema#isEmbargoed":"embargoStatus", - "http://purl.org/spar/pso/Status":"embargoStatus", - "http://vocab.ox.ac.uk/dataset/schema#embargoedUntil":"embargoedUntilDate", - "http://purl.org/spar/fabio/hasEmbargoDate":"embargoedUntilDate", - "http://vocab.ox.ac.uk/dataset/schema#currentVersion":"currentVersion", - "http://purl.org/ontology/bibo/doi":"doi", - "http://www.openarchives.org/ore/terms/aggregates":"aggregatedResource" - "http://purl.org/spar/fabio/publicationDate":"publicationDate", - "http://dublincore.org/documents/dcmi-terms/abstract":"abstract", - "http://dublincore.org/documents/dcmi-terms/accessRights":"accessRights", - "http://dublincore.org/documents/dcmi-terms/accrualMethod":"accrualMethod", - "http://dublincore.org/documents/dcmi-terms/accrualPeriodicity":"accrualPeriodicity", - "http://dublincore.org/documents/dcmi-terms/accrualPolicy":"accrualPolicy", - "http://dublincore.org/documents/dcmi-terms/alternative":"alternative", - "http://dublincore.org/documents/dcmi-terms/audience":"audience", - "http://dublincore.org/documents/dcmi-terms/available":"available", - "http://dublincore.org/documents/dcmi-terms/bibliographicCitation":"bibliographicCitation", - "http://dublincore.org/documents/dcmi-terms/conformsTo":"conformsTo", - "http://dublincore.org/documents/dcmi-terms/contributor":"contributor", - "http://dublincore.org/documents/dcmi-terms/coverage":"coverage", - "http://dublincore.org/documents/dcmi-terms/created":"created", - "http://dublincore.org/documents/dcmi-terms/creator":"creator", - "http://dublincore.org/documents/dcmi-terms/date":"date", - "http://dublincore.org/documents/dcmi-terms/dateAccepted":"dateAccepted", - "http://dublincore.org/documents/dcmi-terms/dateCopyrighted":"dateCopyrighted", - "http://dublincore.org/documents/dcmi-terms/dateSubmitted":"dateSubmitted", - "http://dublincore.org/documents/dcmi-terms/description":"description", - "http://dublincore.org/documents/dcmi-terms/educationLevel":"educationLevel", - "http://dublincore.org/documents/dcmi-terms/extent":"extent", - "http://dublincore.org/documents/dcmi-terms/format":"format", - "http://dublincore.org/documents/dcmi-terms/hasFormat":"hasFormat", - "http://dublincore.org/documents/dcmi-terms/hasPart":"hasPart", - "http://dublincore.org/documents/dcmi-terms/hasVersion":"hasVersion", - "http://dublincore.org/documents/dcmi-terms/identifier":"identifier", - "http://dublincore.org/documents/dcmi-terms/instructionalMethod":"instructionalMethod", - "http://dublincore.org/documents/dcmi-terms/isFormatOf":"isFormatOf", - "http://dublincore.org/documents/dcmi-terms/isPartOf":"isPartOf", - "http://dublincore.org/documents/dcmi-terms/isReferencedBy":"isReferencedBy", - "http://dublincore.org/documents/dcmi-terms/isReplacedBy":"isReplacedBy", - "http://dublincore.org/documents/dcmi-terms/isRequiredBy":"isRequiredBy", - "http://dublincore.org/documents/dcmi-terms/issued":"issued", - "http://dublincore.org/documents/dcmi-terms/isVersionOf":"isVersionOf", - "http://dublincore.org/documents/dcmi-terms/language":"language", - "http://dublincore.org/documents/dcmi-terms/license":"license", - "http://dublincore.org/documents/dcmi-terms/mediator":"mediator", - "http://dublincore.org/documents/dcmi-terms/medium":"medium", - "http://dublincore.org/documents/dcmi-terms/modified":"modified", - "http://dublincore.org/documents/dcmi-terms/provenance":"provenance", - "http://dublincore.org/documents/dcmi-terms/publisher":"publisher", - "http://dublincore.org/documents/dcmi-terms/references":"references", - "http://dublincore.org/documents/dcmi-terms/relation":"relation", - "http://dublincore.org/documents/dcmi-terms/replaces":"replaces", - "http://dublincore.org/documents/dcmi-terms/requires":"requires", - "http://dublincore.org/documents/dcmi-terms/rights":"rights", - "http://dublincore.org/documents/dcmi-terms/rightsHolder":"rightsHolder", - "http://dublincore.org/documents/dcmi-terms/source":"source", - "http://dublincore.org/documents/dcmi-terms/spatial":"spatial", - "http://dublincore.org/documents/dcmi-terms/subject":"subject", - "http://dublincore.org/documents/dcmi-terms/tableOfContents":"tableOfContents", - "http://dublincore.org/documents/dcmi-terms/temporal":"temporal", - "http://dublincore.org/documents/dcmi-terms/title":"title", - "http://dublincore.org/documents/dcmi-terms/type":"type", - "http://dublincore.org/documents/dcmi-terms/valid":"valid", - "http://purl.org/dc/elements/1.1/contributor":"contributor", - "http://purl.org/dc/elements/1.1/coverage":"coverage", - "http://purl.org/dc/elements/1.1/creator":"creator", - "http://purl.org/dc/elements/1.1/date":"date", - "http://purl.org/dc/elements/1.1/description":"description", - "http://purl.org/dc/elements/1.1/format":"format", - "http://purl.org/dc/elements/1.1/identifier":"identifier", - "http://purl.org/dc/elements/1.1/language":"language", - "http://purl.org/dc/elements/1.1/publisher":"publisher", - "http://purl.org/dc/elements/1.1/relation":"relation", - "http://purl.org/dc/elements/1.1/rights":"rights", - "http://purl.org/dc/elements/1.1/source":"source", - "http://purl.org/dc/elements/1.1/subject":"subject", - "http://purl.org/dc/elements/1.1/title":"title", - "http://purl.org/dc/elements/1.1/type":"type" -} - -facets = [ - 'f_creator', - 'f_mediator', - 'f_embargoedUntilDate', - 'f_license', - 'f_rights', - 'f_type', - 'f_publisher', - 'f_isPartOf', - 'f_hasVersion', - 'f_publicationDate', - 'f_contributor', - 'f_language', - 'f_rightsHolder', - 'f_source', - 'f_subject' -] \ No newline at end of file diff --git a/docs/indexingSolrRecordsUsingSupervisord/solr_worker.py b/docs/indexingSolrRecordsUsingSupervisord/solr_worker.py deleted file mode 100644 index 32bebab..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/solr_worker.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python - -from redisqueue import RedisQueue -from LogConfigParser import Config -from solrFields import solr_fields_mapping - -import sys -from time import sleep -from rdflib import URIRef -import simplejson - -from recordsilo import Granary -from solr import SolrConnection - -DB_ROOT = "/opt/RDFDatabank" - -class NoSuchSilo(Exception): - pass - -def gather_document(silo_name, item): - graph = item.get_graph() - document = {'uuid':item.metadata['uuid'], 'id':item.item_id, 'silo':silo_name} - for (_,p,o) in graph.triples((URIRef(item.uri), None, None)): - #print p,o - #if p.startswith("http://purl.org/dc/terms/"): - if p in solr_fields_mapping: - #field = p[len("http://purl.org/dc/terms/"):].encode("utf-8") - field = solr_fields_mapping[p] - if not document.has_key(field): - document[field] = [] - #print "Adding %s to %s" % (o, field) - document[field].append(unicode(o).encode("utf-8")) - else: - if not document.has_key('text'): - document['text'] = [] - #print "Adding %s to %s - (from %s)" % (o, "text", p) - document['text'].append(unicode(o).encode("utf-8")) - #print document - return document - -if __name__ == "__main__": - c = Config() - redis_section = "redis" - worker_section = "worker_solr" - worker_number = sys.argv[1] - if len(sys.argv) == 3: - if "redis_%s" % sys.argv[2] in c.sections(): - redis_section = "redis_%s" % sys.argv[2] - - rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number, - db=c.get(redis_section, "db"), - host=c.get(redis_section, "host"), - port=c.get(redis_section, "port") - ) - rdfdb_config = Config("%s/production.ini" % DB_ROOT) - granary_root = rdfdb_config.get("app:main", "granary.store", 0, {'here':DB_ROOT}) - - g = Granary(granary_root) - - solr = SolrConnection(c.get(worker_section, "solrurl")) - - idletime = 2 - - while(True): - sleep(idletime) - line = rq.pop() - if not line: - continue - msg = simplejson.loads(line) - # solr switch - silo_name = msg['silo'] - if silo_name not in g.silos: - raise NoSuchSilo - s = g.get_rdf_silo(silo_name) - if msg['type'] == "c" or msg['type'] == "u" or msg['type'] == "embargo": - # Creation - itemid = msg.get('id') - print "Got creation message on id:%s in silo:%s" % (itemid, silo_name) - if itemid and s.exists(itemid): - item = s.get_item(itemid) - solr_doc = gather_document(silo_name, item) - try: - solr.add(_commit=True, **solr_doc) - except: - f = open('/opt/logs/solr_error.log', 'a') - f.write("Error adding record (creating) id:%s in silo:%s" % (itemid, silo_name)) - f.close() - rq.task_complete() - elif msg['type'] == "d": - # Deletion - itemid = msg.get('id') - if itemid and s.exists(itemid): - solr.delete(itemid) - solr.commit() - rq.task_complete() - """ - elif msg['type'] == "embargo": - itemid = msg.get('id') - if itemid and s.exists(itemid): - if msg['embargoed'] in ['false', 'False', 0, False]: - # Embargo removed: update solr - item = s.get_item(itemid) - solr_doc = gather_document(silo_name, item) - try: - solr.add(_commit=True, **solr_doc) - except: - f = open('/opt/logs/solr_error.log', 'a') - f.write("Error adding record (embargo change) id:%s in silo:%s" % (itemid, silo_name)) - f.close() - else: - solr.delete(itemid) - solr.commit() - rq.task_complete() - except NoSuchSilo: - print "ERROR: Silo doesn't exist %s" % silo_name - print line - rq.task_complete() - """ diff --git a/docs/indexingSolrRecordsUsingSupervisord/supervisor_startup.sh b/docs/indexingSolrRecordsUsingSupervisord/supervisor_startup.sh deleted file mode 100644 index 667f8fb..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/supervisor_startup.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -su databankadmin --command "exec /usr/local/bin/supervisord -c /opt/workers/supervisord.conf" - diff --git a/docs/indexingSolrRecordsUsingSupervisord/supervisord.conf b/docs/indexingSolrRecordsUsingSupervisord/supervisord.conf deleted file mode 100644 index 098e111..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/supervisord.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -username = guest -password = guest -file = /tmp/supervisor.sock - -[supervisord] -minfds = 1024 -minprocs = 200 -loglevel = info -logfile = /tmp/supervisord.log -logfile_maxbytes = 50MB -nodaemon = false -pidfile = /tmp/supervisord.pid -logfile_backups = 10 - -[supervisorctl] -serverurl = unix:///tmp/supervisor.sock - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[include] -files = workers_enabled/*.conf - -[inet_http_server] -username = guest -password = guest -port = 127.0.0.1:9001 - diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_articlelogger.conf b/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_articlelogger.conf deleted file mode 100644 index 30b3481..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_articlelogger.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:logger_articlelogger] -autorestart = true -numprocs = 1 -startretries = 3 -redirect_stderr = True -stopwaitsecs = 10 -process_name = logger_articlelogger_%(process_num)s -priority = 999 -command = ./logfromqueue.py %(process_num)s logger_articlelogger -autostart = true -startsecs = 10 -stdout_logfile = workerlogs/articlelogger.log - diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_auditlogger.conf b/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_auditlogger.conf deleted file mode 100644 index 65c219d..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/logger_auditlogger.conf +++ /dev/null @@ -1,13 +0,0 @@ -[program:logger_auditlogger] -autorestart = true -numprocs = 1 -startretries = 3 -redirect_stderr = True -stopwaitsecs = 10 -process_name = logger_auditlogger_%(process_num)s -priority = 666 -directory = /opt/workers/ -command = /opt/workers/logfromqueue.py %(process_num)s logger_auditlogger -autostart = true -startsecs = 10 -stdout_logfile = /opt/workers/workerlogs/auditlogger.log diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_redis.conf b/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_redis.conf deleted file mode 100644 index b024366..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_redis.conf +++ /dev/null @@ -1,14 +0,0 @@ -[program:process_redis] -autorestart = true -numprocs = 1 -startretries = 3 -redirect_stderr = True -stopwaitsecs = 10 -process_name = process_redis_%(process_num)s -priority = 555 -directory = /opt/workers/ -command = /opt/workers/redis-1.2.6/redis-server -autostart = true -startsecs = 10 -stdout_logfile = /opt/workers/workerlogs/process_redis.log - diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_solr.conf b/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_solr.conf deleted file mode 100644 index c866f22..0000000 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/process_solr.conf +++ /dev/null @@ -1,14 +0,0 @@ -[program:process_solr] -autorestart = true -numprocs = 1 -startretries = 3 -redirect_stderr = True -stopwaitsecs = 10 -process_name = process_solr_%(process_num)s -priority = 999 -directory = /opt/solr/ -command = /opt/solr/start_solr.sh -autostart = true -startsecs = 10 -stdout_logfile = /opt/workers/workerlogs/process_solr.log - diff --git a/docs/solr_config/conf/schema.xml b/docs/solr_config/conf/schema.xml index 410d98d..97e0e99 100644 --- a/docs/solr_config/conf/schema.xml +++ b/docs/solr_config/conf/schema.xml @@ -262,64 +262,64 @@ - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/using_databank_api/DatabankDemo.py b/docs/using_databank_api/DatabankDemo.py new file mode 100644 index 0000000..4045c17 --- /dev/null +++ b/docs/using_databank_api/DatabankDemo.py @@ -0,0 +1,78 @@ +#-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +#Databank API demo + +import urllib2 +import base64 +import urllib +from multipartform import MultiPartForm + +#=============================================================================== +#Using urllib2 to create a package in Databank +url = "http://databank-vm1.oerc.ox.ac.uk/test/datasets" +req = urllib2.Request(url) +USER = "admin" +PASS = "test" +identifier = "TestSubmission" +auth = 'Basic ' + base64.urlsafe_b64encode("%s:%s" % (USER, PASS)) +req.add_header('Authorization', auth) +req.add_header('Accept', 'application/JSON') +req.add_data(urllib.urlencode({'id': identifier})) + +# To verify the method is POST +req.get_method() + +ans = urllib2.urlopen(req) + +ans.read() +ans.msg +ans.code + +#=============================================================================== +#Using urllib2 to post a file in Databank +#Add a file +form = MultiPartForm() +filename = "solrconfig.xml" +filepath = "./solrconfig.xml" +form.add_file('file', filename, fileHandle=open(filepath)) + +# Build the request +url2 = "http://databank-vm1.oerc.ox.ac.uk/test/datasets/TestSubmission" +req2 = urllib2.Request(url2) +auth = 'Basic ' + base64.urlsafe_b64encode("admin:test") +req2.add_header('Authorization', auth) +req2.add_header('Accept', 'application/JSON') +body = str(form) +req2.add_header('Content-type', form.get_content_type()) +req2.add_header('Content-length', len(body)) +req2.add_data(body) + +print +print 'OUTGOING DATA:' +print req2.get_data() +ans2 = urllib2.urlopen(req2) +print +print 'SERVER RESPONSE:' +ans2.read() \ No newline at end of file diff --git a/docs/using_databank_api/HTTP_request.py b/docs/using_databank_api/HTTP_request.py index 796a1f9..bd76624 100644 --- a/docs/using_databank_api/HTTP_request.py +++ b/docs/using_databank_api/HTTP_request.py @@ -1,3 +1,27 @@ +#-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import mimetypes import httplib diff --git a/docs/using_databank_api/main.py b/docs/using_databank_api/main.py index 6872808..d9f7748 100644 --- a/docs/using_databank_api/main.py +++ b/docs/using_databank_api/main.py @@ -1,3 +1,26 @@ +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + # Using the databank API """ @@ -6,7 +29,7 @@ The full functionality of RDFDatabank is detailed in the API documentation at http://databank.ora.ox.ac.uk/api -https://github.com/anusharanganathan/RDFDatabank/tree/master/rdfdatabank/public/static/api_files +https://github.com/dataflow/RDFDatabank/tree/master/rdfdatabank/public/static/api_files """ import json as simplejson diff --git a/docs/using_databank_api/multipart.py b/docs/using_databank_api/multipart.py new file mode 100644 index 0000000..14aa3e7 --- /dev/null +++ b/docs/using_databank_api/multipart.py @@ -0,0 +1,78 @@ +# --------------------------------------------------------------------- +# +# Copyright (c) 2012 University of Oxford +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# --------------------------------------------------------------------- + +import mimetools +import mimetypes + +class MultiPartFormData(object): + def __init__(self, fields=None, files=None): + self._boundary = mimetools.choose_boundary() + self._fields = fields or () + self._files = files or () + for file in self._files: + file['mimetype'] = file.get('mimetype') or mimetypes.guess_type(file['filename'])[0] or 'application/octet-stream' + self._body = self._body_iterator() + + @property + def content_type(self): + return 'multipart/form-data; boundary=%s' % self._boundary + + @property + def content_length(self): + field_padding = '--\r\nContent-Disposition: form-data; name=""\r\n\r\n\r\n' + file_padding = '--\r\nContent-Disposition: form-data; name=""; filename=""\r\nContent-Type: \r\n\r\n' + + field_length = sum(sum(map(len, [self._boundary, field_padding, k, v])) for k,v in self._fields) + file_length = sum(f['size'] + sum(map(len, [self._boundary, file_padding, f['name'], f['filename'], f['mimetype']])) for f in self._files) + + return field_length + file_length + len('----\r\n') + len(self._boundary) + + def _body_iterator(self): + for (key, value) in self._fields: + yield '--%s\r\n' % self._boundary + yield 'Content-Disposition: form-data; name="%s"\r\n' % key + yield '\r\n' + if value: + yield value + yield '\r\n' + for file in self._files: + yield '--%s\r\n' % self._boundary + yield 'Content-Disposition: form-data; name="%s"; filename="%s"\r\n' % (file['name'], file['filename']) + yield 'Content-Type: %s\r\n' % file['mimetype'] + yield '\r\n' + + stream = file['stream'] + while True: + data = stream.read(4096) + if not data: + break + yield data + yield '--%s--\r\n' % self._boundary + + def read(self, blocksize): + try: + return self._body.next() + except StopIteration: + return '' diff --git a/docs/using_databank_api/multipartform.py b/docs/using_databank_api/multipartform.py new file mode 100644 index 0000000..0bfd916 --- /dev/null +++ b/docs/using_databank_api/multipartform.py @@ -0,0 +1,90 @@ +#-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import itertools +import mimetools +import mimetypes + +class MultiPartForm(object): + """Accumulate the data to be used when posting a form.""" + + def __init__(self): + self.form_fields = [] + self.files = [] + self.boundary = mimetools.choose_boundary() + return + + def get_content_type(self): + return 'multipart/form-data; boundary=%s' % self.boundary + + def add_field(self, name, value): + """Add a simple field to the form data.""" + self.form_fields.append((name, value)) + return + + def add_file(self, fieldname, filename, fileHandle, mimetype=None): + """Add a file to be uploaded.""" + body = fileHandle.read() + if mimetype is None: + mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + self.files.append((fieldname, filename, mimetype, body)) + return + + def __str__(self): + """Return a string representing the form data, including attached files.""" + # Build a list of lists, each containing "lines" of the + # request. Each part is separated by a boundary string. + # Once the list is built, return a string where each + # line is separated by '\r\n'. + parts = [] + part_boundary = '--' + self.boundary + + # Add the form fields + parts.extend( + [ part_boundary, + 'Content-Disposition: form-data; name="%s"' % name, + '', + value, + ] + for name, value in self.form_fields + ) + + # Add the files to upload + parts.extend( + [ part_boundary, + 'Content-Disposition: file; name="%s"; filename="%s"' % \ + (field_name, filename), + 'Content-Type: %s' % content_type, + '', + body, + ] + for field_name, filename, content_type, body in self.files + ) + + # Flatten the list and add closing boundary marker, + # then return CR+LF separated data + flattened = list(itertools.chain(*parts)) + flattened.append('--' + self.boundary + '--') + flattened.append('') + return '\r\n'.join(flattened) \ No newline at end of file diff --git a/docs/using_databank_api/postingToDatabank.py b/docs/using_databank_api/postingToDatabank.py new file mode 100644 index 0000000..89326b3 --- /dev/null +++ b/docs/using_databank_api/postingToDatabank.py @@ -0,0 +1,89 @@ +#-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import urllib2 +import base64 +import urllib +from multipart import MultiPartFormData +import os + +#=============================================================================== +#Using urllib2 to create a package in Databank +url = "http://databank-vm1.oerc.ox.ac.uk/test/datasets" +identifier = "TestSubmission" +req = urllib2.Request(url) +USER = "admin" +PASS = "test" +auth = 'Basic ' + base64.urlsafe_b64encode("%s:%s" % (USER, PASS)) +req.add_header('Authorization', auth) +req.add_header('Accept', 'application/JSON') +req.add_data(urllib.urlencode({'id': identifier})) + +# To verify the method is POST +req.get_method() + +ans = urllib2.urlopen(req) + +ans.read() +ans.msg +ans.code + +#=============================================================================== +#Using urllib2 to post a file +#Add a file + +filename = "solrconfig.xml" +filepath = "./solrconfig.xml" +f = open(filepath, 'rb') +stat_info = os.stat(filepath) + +file1_info = { + 'name':'file', + 'filename':filename, + 'mimetype':'application/xml', + 'stream': f, + 'size':int(stat_info.st_size)} + +data = MultiPartFormData(files=[file1_info]) + +# Build the request +url2 = "http://databank-vm1.oerc.ox.ac.uk/test/datasets/TestSubmission" +req2 = urllib2.Request(url2) +auth = 'Basic ' + base64.urlsafe_b64encode("admin:test") +req2.add_header('Authorization', auth) +req2.add_header('Accept', 'application/JSON') +req2.add_header('Content-type', data.content_type) +req2.add_header('Content-length', data.content_length) + +body = ''.join(list(data._body)) +req2.add_data(str(body)) + +#print +#print 'OUTGOING DATA:' +#print req2.get_data() +ans2 = urllib2.urlopen(req2) +#print +print 'SERVER RESPONSE:' +ans2.read() +#=============================================================================== \ No newline at end of file diff --git a/docs/using_zipfile_library.py b/docs/using_zipfile_library.py deleted file mode 100644 index 9b31e8c..0000000 --- a/docs/using_zipfile_library.py +++ /dev/null @@ -1,123 +0,0 @@ -from zipfile import ZipFile, BadZipfile as BZ -#================================================ -def read_zipfile(filepath): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - zipfile_contents = {} - for info in tmpfile.infolist(): - zipfile_contents[info.filename] = (info.file_size, info.date_time) - tmpfile.close() - return zipfile_contents -#================================================ -def read_file_in_zipfile(filepath, filename): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - try: - fileinfo = tmpfile.getinfo(filename) - except KeyError: - return False - if fileinfo.file_size == 0: - return 0 - file_contents = None - file_contents = tmpfile.read(filename) - tmpfile.close() - return file_contents -#================================================ -def get_file_in_zipfile(filepath, filename, targetdir): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - try: - fileinfo = tmpfile.getinfo(filename) - except KeyError: - return False - if fileinfo.file_size == 0: - return 0 - targetfile = tmpfile.extract(filename, targetdir) - tmpfile.close() - return targetfile -#================================================ -path = 'silos/sandbox/pairtree_root/da/ta/se/t1/obj/__26/' -fp1 = path + 'test3.zip' -fp2 = path + 'read_test.zip' -fp3 = path + 'databank_logo.png' - -zc1 = read_zipfile(fp1) -zc2 = read_zipfile(fp2) -zc3 = read_zipfile(fp3) - -zc1_files = zc1.keys() -zc2_files = zc2.keys() - -ans11 = read_file_in_zipfile(fp1, zc1_files[1]) #expected: 0 -ans12 = read_file_in_zipfile(fp1, 'test') #expected: False -ans13 = read_file_in_zipfile(fp1, zc1_files[0]) #expected: file conts - -ans21 = read_file_in_zipfile(fp2, zc2_files[0]) #expected: file conts -ans22 = read_file_in_zipfile(fp2, zc2_files[1]) #expected: 0 -ans23 = read_file_in_zipfile(fp2, zc2_files[4]) #expected: binary output - -ans14 = get_file_in_zipfile(fp1, zc1_files[1], '/tmp') #expected: 0 -ans15 = get_file_in_zipfile(fp1, 'test', '/tmp') #expected: False -ans16 = get_file_in_zipfile(fp1, zc1_files[0], '/tmp') #expected: '/tmp/admiral-dataset.txt' - -ans24 = get_file_in_zipfile(fp2, zc2_files[0], '/tmp') #expected: '/tmp/read_test/Dir/TestScanFiles32.txt' -ans25 = get_file_in_zipfile(fp2, zc2_files[1], '/tmp') #expected: 0 -ans26 = get_file_in_zipfile(fp2, zc2_files[4], '/tmp') #expected: '/tmp/read_test/databank_logo.png' -#================================================ -#Expected Answers -""" ->>> zc1 -{'admiral-dataset.txt': (43, (2010, 11, 29, 16, 30, 52)), 'TestScanFilesSubDir/': (0, (2010, 11, 29, 17, 34, 42)), 'TestScanFilesSubDir/TestScanFiles31.txt': (9, (2010, 10, 4, 15, 39, 54)), 'TestScanFilesSubDir/TestScanFiles32.txt': (9, (2010, 10, 4, 15, 39, 54)), 'TestScanFilesSubDir/manifest.rdf': (511, (2010, 11, 29, 17, 42, 10))} - ->>> zc2 -{'read_test/Dir/TestScanFiles32.txt': (9, (2010, 10, 4, 15, 39, 54)), 'read_test/Dir/': (0, (2011, 1, 5, 13, 43, 30)), 'read_test/admiral-dataset.txt': (43, (2010, 11, 29, 16, 30, 52)), 'read_test/Dir/manifest.rdf': (511, (2010, 11, 29, 17, 42, 10)), 'read_test/databank_logo.png': (20220, (2010, 12, 6, 15, 11, 40)), 'read_test/': (0, (2011, 1, 5, 13, 44, 40)), 'read_test/Dir/TestScanFiles31.txt': (9, (2010, 10, 4, 15, 39, 54))} - ->>> zc1_files -['admiral-dataset.txt', 'TestScanFilesSubDir/', 'TestScanFilesSubDir/TestScanFiles31.txt', 'TestScanFilesSubDir/TestScanFiles32.txt', 'TestScanFilesSubDir/manifest.rdf'] - ->>> zc2_files -['read_test/Dir/TestScanFiles32.txt', 'read_test/Dir/', 'read_test/admiral-dataset.txt', 'read_test/Dir/manifest.rdf', 'read_test/databank_logo.png', 'read_test/', 'read_test/Dir/TestScanFiles31.txt'] - ->>> ans11 -0 - ->>> ans12 -False - ->>> ans13 -'This directory contains an ADMIRAL dataset\n' - ->>> ans21 -'Test file' - ->>> ans22 -0 - ->>> ans23 -'\x89PNG\..... - ->>> ans14 -0 - ->>> ans15 -False - ->>> ans16 -'/tmp/admiral-dataset.txt' - ->>> ans24 -'/tmp/read_test/Dir/TestScanFiles32.txt' - ->>> ans25 -0 - ->>> ans26 -'/tmp/read_test/databank_logo.png' -""" -#================================================ diff --git a/message_workers/LogConfigParser.py b/message_workers/LogConfigParser.py new file mode 100755 index 0000000..92b8531 --- /dev/null +++ b/message_workers/LogConfigParser.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import ConfigParser, os + +class Config(ConfigParser.ConfigParser): + DEFAULT_CONFIG_FILE = "loglines.cfg" + def __init__(self, config_file=DEFAULT_CONFIG_FILE): + ConfigParser.ConfigParser.__init__(self) + if os.path.exists(config_file) and os.path.isfile(config_file): + self.read(config_file) + self.validate() + + def validate(self): + pass diff --git a/docs/indexingSolrRecordsUsingSupervisord/broker.py b/message_workers/broker.py old mode 100644 new mode 100755 similarity index 64% rename from docs/indexingSolrRecordsUsingSupervisord/broker.py rename to message_workers/broker.py index 20a5f6c..6746857 --- a/docs/indexingSolrRecordsUsingSupervisord/broker.py +++ b/message_workers/broker.py @@ -1,4 +1,27 @@ -#!/usr/bin/env python +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" from redisqueue import RedisQueue @@ -31,13 +54,13 @@ port=c.get(redis_section, "port") ) fanout_queues = [x.strip() for x in c.get(worker_section, "fanout").split(",") if x] - + if c.has_option(worker_section, "idletime"): try: idletime = float(c.get(worker_section, "idletime")) except ValueError: idletime = 10 - + while(True): line = rq.pop() if line: diff --git a/message_workers/loglines.cfg b/message_workers/loglines.cfg new file mode 100644 index 0000000..5399fbf --- /dev/null +++ b/message_workers/loglines.cfg @@ -0,0 +1,58 @@ +#Copyright (c) 2012 University of Oxford +# +#Permission is hereby granted, free of charge, to any person obtaining +#a copy of this software and associated documentation files (the +#"Software"), to deal in the Software without restriction, including +#without limitation the rights to use, copy, modify, merge, publish, +#distribute, sublicense, and/or sell copies of the Software, and to +#permit persons to whom the Software is furnished to do so, subject to +#the following conditions: +# +#The above copyright notice and this permission notice shall be +#included in all copies or substantial portions of the Software. +# +#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +#EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +#MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +#IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +#CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +#TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +#SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# Configuring workers: +[worker_solr] +listento = solrindex +command = ./solr_worker.py +solrurl = http://localhost:8080/solr +dbroot = /var/lib/databank +idletime = 1 +stdout_logfile = workerlogs/solr_worker.log +numprocs = 2 + +[worker_broker] +listento = silochanges +command = ./broker.py +#fanout = auditlog, solrindex +fanout = solrindex +fanout_status_queue = broker_temp +idletime = 1 +stdout_logfile = workerlogs/broker.log +numprocs = 2 + +[logger_auditlogger] +listento = auditlog +command = ./logfromqueue.py +logfile = logs/audit.log +stdout_logfile = workerlogs/auditlogger.log + +# DEFAULT VALUES FOLLOW +############################## +[redis] +host = localhost +port = 6379 +db = 0 + +[redis_test] +host = localhost +port = 6379 +db = 1 diff --git a/docs/indexingSolrRecordsUsingSupervisord/redisqueue.py b/message_workers/redisqueue.py old mode 100644 new mode 100755 similarity index 62% rename from docs/indexingSolrRecordsUsingSupervisord/redisqueue.py rename to message_workers/redisqueue.py index d3e422b..79d38a6 --- a/docs/indexingSolrRecordsUsingSupervisord/redisqueue.py +++ b/message_workers/redisqueue.py @@ -1,6 +1,31 @@ -#!/usr/bin/env python +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" from redis import Redis +from redis.exceptions import ConnectionError +from time import sleep WORKERPREFIX = "temp" HOST = "localhost" @@ -45,6 +70,13 @@ def _initclient(self): logger.debug("Debug messages detailing worker queue activity") self._r = Redis(host=self.host, db=self.db, port=self.port) + def check_connection(self): + #sleep(1) + try: + self._r.info() + except ConnectionError: + self._initclient() + def __len__(self): if self.inprogress(): return self._r.llen(self.queuename) + 1 @@ -52,9 +84,12 @@ def __len__(self): return self._r.llen(self.queuename) def __getitem__(self, index): + #self.check_connection() return self._r.lrange(self.queuename, index, index) def inprogress(self): + #sleep(1) + #self.check_connection() ip = self._r.lrange(self.workeritem, 0, 0) if ip: return ip.pop() @@ -62,15 +97,21 @@ def inprogress(self): return None def task_complete(self): + #sleep(1) + #self.check_connection() logger.debug("Task completed by worker %s" % self.workername) return self._r.rpop(self.workeritem) def task_failed(self): + #sleep(1) + #self.check_connection() logger.error("Task FAILED by worker %s" % self.workername) logger.debug(self.inprogress()) return self._r.rpoplpush(self.workeritem, self.queuename) def push(self, item, to_queue=None): + #sleep(1) + #self.check_connection() if to_queue: logger.debug("{%s} put onto queue %s by worker %s" % (item, to_queue,self.workername)) return self._r.lpush(to_queue, item) @@ -79,6 +120,9 @@ def push(self, item, to_queue=None): return self._r.lpush(self.queuename, item) def pop(self): + #sleep(1) + #self.check_connection() + logger.debug("In pop - Queuename: %s, workeritem:%s"%(self.queuename, self.workeritem)) if self._r.llen(self.workeritem) == 0: self._r.rpoplpush(self.queuename, self.workeritem) logger.debug("{%s} pulled from queue %s by worker %s" % (self.inprogress(), self.queuename,self.workername)) diff --git a/message_workers/solrFields.py b/message_workers/solrFields.py new file mode 100644 index 0000000..45dffe8 --- /dev/null +++ b/message_workers/solrFields.py @@ -0,0 +1,127 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +solr_fields_mapping = { + "silo":"silo", + "id":"id", + "uuid":"uuid", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#type":"type", + "http://vocab.ox.ac.uk/dataset/schema#isEmbargoed":"embargoStatus", + "http://purl.org/spar/pso/Status":"embargoStatus", + "http://vocab.ox.ac.uk/dataset/schema#embargoedUntil":"embargoedUntilDate", + "http://purl.org/spar/fabio/hasEmbargoDate":"embargoedUntilDate", + "http://vocab.ox.ac.uk/dataset/schema#currentVersion":"currentVersion", + "http://purl.org/ontology/bibo/doi":"doi", + "http://www.openarchives.org/ore/terms/aggregates":"aggregatedResource", + "http://purl.org/spar/fabio/publicationDate":"publicationDate", + "http://purl.org/dc/terms/abstract":"abstract", + "http://purl.org/dc/terms/accessRights":"accessRights", + "http://purl.org/dc/terms/accrualMethod":"accrualMethod", + "http://purl.org/dc/terms/accrualPeriodicity":"accrualPeriodicity", + "http://purl.org/dc/terms/accrualPolicy":"accrualPolicy", + "http://purl.org/dc/terms/alternative":"alternative", + "http://purl.org/dc/terms/audience":"audience", + "http://purl.org/dc/terms/available":"available", + "http://purl.org/dc/terms/bibliographicCitation":"bibliographicCitation", + "http://purl.org/dc/terms/conformsTo":"conformsTo", + "http://purl.org/dc/terms/contributor":"contributor", + "http://purl.org/dc/terms/coverage":"coverage", + "http://purl.org/dc/terms/created":"created", + "http://purl.org/dc/terms/creator":"creator", + "http://purl.org/dc/terms/date":"date", + "http://purl.org/dc/terms/dateAccepted":"dateAccepted", + "http://purl.org/dc/terms/dateCopyrighted":"dateCopyrighted", + "http://purl.org/dc/terms/dateSubmitted":"dateSubmitted", + "http://purl.org/dc/terms/description":"description", + "http://purl.org/dc/terms/educationLevel":"educationLevel", + "http://purl.org/dc/terms/extent":"extent", + "http://purl.org/dc/terms/format":"format", + "http://purl.org/dc/terms/hasFormat":"hasFormat", + "http://purl.org/dc/terms/hasPart":"hasPart", + "http://purl.org/dc/terms/hasVersion":"hasVersion", + "http://purl.org/dc/terms/identifier":"identifier", + "http://purl.org/dc/terms/instructionalMethod":"instructionalMethod", + "http://purl.org/dc/terms/isFormatOf":"isFormatOf", + "http://purl.org/dc/terms/isPartOf":"isPartOf", + "http://purl.org/dc/terms/isReferencedBy":"isReferencedBy", + "http://purl.org/dc/terms/isReplacedBy":"isReplacedBy", + "http://purl.org/dc/terms/isRequiredBy":"isRequiredBy", + "http://purl.org/dc/terms/issued":"issued", + "http://purl.org/dc/terms/isVersionOf":"isVersionOf", + "http://purl.org/dc/terms/language":"language", + "http://purl.org/dc/terms/license":"license", + "http://purl.org/dc/terms/mediator":"mediator", + "http://purl.org/dc/terms/medium":"medium", + "http://purl.org/dc/terms/modified":"modified", + "http://purl.org/dc/terms/provenance":"provenance", + "http://purl.org/dc/terms/publisher":"publisher", + "http://purl.org/dc/terms/references":"references", + "http://purl.org/dc/terms/relation":"relation", + "http://purl.org/dc/terms/replaces":"replaces", + "http://purl.org/dc/terms/requires":"requires", + "http://purl.org/dc/terms/rights":"rights", + "http://purl.org/dc/terms/rightsHolder":"rightsHolder", + "http://purl.org/dc/terms/source":"source", + "http://purl.org/dc/terms/spatial":"spatial", + "http://purl.org/dc/terms/subject":"subject", + "http://purl.org/dc/terms/tableOfContents":"tableOfContents", + "http://purl.org/dc/terms/temporal":"temporal", + "http://purl.org/dc/terms/title":"title", + "http://purl.org/dc/terms/type":"type", + "http://purl.org/dc/terms/valid":"valid", + "http://purl.org/dc/elements/1.1/contributor":"contributor", + "http://purl.org/dc/elements/1.1/coverage":"coverage", + "http://purl.org/dc/elements/1.1/creator":"creator", + "http://purl.org/dc/elements/1.1/date":"date", + "http://purl.org/dc/elements/1.1/description":"description", + "http://purl.org/dc/elements/1.1/format":"format", + "http://purl.org/dc/elements/1.1/identifier":"identifier", + "http://purl.org/dc/elements/1.1/language":"language", + "http://purl.org/dc/elements/1.1/publisher":"publisher", + "http://purl.org/dc/elements/1.1/relation":"relation", + "http://purl.org/dc/elements/1.1/rights":"rights", + "http://purl.org/dc/elements/1.1/source":"source", + "http://purl.org/dc/elements/1.1/subject":"subject", + "http://purl.org/dc/elements/1.1/title":"title", + "http://purl.org/dc/elements/1.1/type":"type" +} + +facets = [ + 'f_creator', + 'f_mediator', + 'f_embargoedUntilDate', + 'f_license', + 'f_rights', + 'f_type', + 'f_publisher', + 'f_isPartOf', + 'f_hasVersion', + 'f_publicationDate', + 'f_contributor', + 'f_language', + 'f_rightsHolder', + 'f_source', + 'f_subject' +] diff --git a/message_workers/solr_worker.py b/message_workers/solr_worker.py new file mode 100755 index 0000000..055056f --- /dev/null +++ b/message_workers/solr_worker.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +from redisqueue import RedisQueue +from LogConfigParser import Config +from solrFields import solr_fields_mapping + +import sys +from time import sleep +from rdflib import URIRef +import simplejson +from collections import defaultdict + +from recordsilo import Granary +from solr import SolrConnection + +import logging + +logger = logging.getLogger("redisqueue") +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") +ch.setFormatter(formatter) +logger.addHandler(ch) + +class NoSuchSilo(Exception): + pass + +def gather_document(silo_name, item): + graph = item.get_graph() + document = defaultdict(list) + document['uuid'].append(item.metadata['uuid']) + document['id'].append(item.item_id) + document['silo'].append(silo_name) + for (_,p,o) in graph.triples((URIRef(item.uri), None, None)): + if str(p) in solr_fields_mapping: + field = solr_fields_mapping[str(p)] + if field == "embargoedUntilDate": + ans = u"%sZ"%unicode(o).split('.')[0] + document[field].append(unicode(ans).encode("utf-8")) + else: + document[field].append(unicode(o).encode("utf-8")) + else: + document['text'].append(unicode(o).encode("utf-8")) + document = dict(document) + return document + +if __name__ == "__main__": + c = Config() + redis_section = "redis" + worker_section = "worker_solr" + worker_number = sys.argv[1] + if len(sys.argv) == 3: + if "redis_%s" % sys.argv[2] in c.sections(): + redis_section = "redis_%s" % sys.argv[2] + + rq = RedisQueue(c.get(worker_section, "listento"), "solr_%s" % worker_number, + db=c.get(redis_section, "db"), + host=c.get(redis_section, "host"), + port=c.get(redis_section, "port") + ) + DB_ROOT = c.get(worker_section, "dbroot") + rdfdb_config = Config("%s/production.ini" % DB_ROOT) + granary_root = rdfdb_config.get("app:main", "granary.store", 0, {'here':DB_ROOT}) + + g = Granary(granary_root) + + solr = SolrConnection(c.get(worker_section, "solrurl")) + + idletime = 2 + + while(True): + sleep(idletime) + line = rq.pop() + if not line: + continue + msg = simplejson.loads(line) + # solr switch + silo_name = msg['silo'] + if silo_name not in g.silos: + raise NoSuchSilo + s = g.get_rdf_silo(silo_name) + if msg['type'] == "c" or msg['type'] == "u" or msg['type'] == "embargo": + # Creation, update or embargo change + itemid = msg.get('id') + logger.info("Got creation message on id:%s in silo:%s" % (itemid, silo_name)) + if itemid and s.exists(itemid): + item = s.get_item(itemid) + solr_doc = gather_document(silo_name, item) + try: + solr.add(_commit=True, **solr_doc) + except Exception, e : + logger.error("Error adding document to solr id:%s in silo:%s\n" % (itemid, silo_name)) + #f = open('/var/log/databank/solr_error.log', 'a') + #f.write("Error adding record (creating) id:%s in silo:%s\n" % (itemid, silo_name)) + try: + logger.error("%s\n\n" %str(e)) + except: + pass + rq.task_complete() + elif msg['type'] == "d": + # Deletion + itemid = msg.get('id') + if itemid and s.exists(itemid): + solr.delete(itemid) + solr.commit() + rq.task_complete() diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_broker.conf b/message_workers/workers_available/worker_broker.conf similarity index 55% rename from docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_broker.conf rename to message_workers/workers_available/worker_broker.conf index d7f8b2b..0323f9a 100644 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_broker.conf +++ b/message_workers/workers_available/worker_broker.conf @@ -6,9 +6,9 @@ redirect_stderr = True stopwaitsecs = 10 process_name = worker_broker_%(process_num)s priority = 777 -directory = /opt/workers/ -command = /opt/workers/broker.py %(process_num)s +directory = /var/lib/databank/message_workers/ +command = /var/lib/databank/message_workers/broker.py %(process_num)s autostart = true startsecs = 10 -stdout_logfile = /opt/workers/workerlogs/broker.log +stdout_logfile = /var/log/databank/broker.log diff --git a/docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_solr.conf b/message_workers/workers_available/worker_solr.conf similarity index 53% rename from docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_solr.conf rename to message_workers/workers_available/worker_solr.conf index a659549..5cb1790 100644 --- a/docs/indexingSolrRecordsUsingSupervisord/workers_available/worker_solr.conf +++ b/message_workers/workers_available/worker_solr.conf @@ -6,9 +6,9 @@ redirect_stderr = True stopwaitsecs = 10 process_name = worker_solr_%(process_num)s priority = 888 -directory = /opt/workers/ -command = /opt/workers/solr_worker.py %(process_num)s +directory = /var/lib/databank/message_workers/ +command = /var/lib/databank/message_workers/solr_worker.py %(process_num)s autostart = true startsecs = 10 -stdout_logfile = /opt/workers/workerlogs/solr_worker.log +stdout_logfile = /var/log/databank/solr_worker.log diff --git a/mod_wsgi/dispatch.wsgi b/mod_wsgi/dispatch.wsgi index 72c4052..67ce816 100644 --- a/mod_wsgi/dispatch.wsgi +++ b/mod_wsgi/dispatch.wsgi @@ -1,16 +1,39 @@ +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + # Add the virtual Python environment site-packages directory to the path #import site #site.addsitedir('/home/simplesite/env/lib/python2.5/site-packages') #site.addsitedir('/usr/local/lib/python2.6/dist-packages') import pkg_resources -pkg_resources.working_set.add_entry('/opt/RDFDatabank') +pkg_resources.working_set.add_entry('/var/lib/databank') # Avoid ``[Errno 13] Permission denied: '/var/www/.python-eggs'`` messages import os -os.environ['PYTHON_EGG_CACHE'] = '/opt/RDFDatabank/egg-cache' +os.environ['PYTHON_EGG_CACHE'] = '/var/cache/databank/egg-cache' # Load the Pylons application from paste.deploy import loadapp -application = loadapp('config:/opt/RDFDatabank/production.ini') +application = loadapp('config:/var/lib/databank/production.ini') diff --git a/mod_wsgi/dispatch_development.wsgi b/mod_wsgi/dispatch_development.wsgi index 0528d21..3b8b054 100644 --- a/mod_wsgi/dispatch_development.wsgi +++ b/mod_wsgi/dispatch_development.wsgi @@ -1,16 +1,39 @@ +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + # Add the virtual Python environment site-packages directory to the path #import site #site.addsitedir('/home/simplesite/env/lib/python2.5/site-packages') #site.addsitedir('/usr/local/lib/python2.6/dist-packages') import pkg_resources -pkg_resources.working_set.add_entry('/opt/RDFDatabank') +pkg_resources.working_set.add_entry('/var/lib/databank') # Avoid ``[Errno 13] Permission denied: '/var/www/.python-eggs'`` messages import os -os.environ['PYTHON_EGG_CACHE'] = '/opt/RDFDatabank/egg-cache' +os.environ['PYTHON_EGG_CACHE'] = '/var/cache/databank/egg-cache' # Load the Pylons application from paste.deploy import loadapp -application = loadapp('config:/opt/RDFDatabank/development.ini') +application = loadapp('config:/var/lib/databank/development.ini') diff --git a/passwd-default b/passwd-default index 6af3641..c4930a9 100644 --- a/passwd-default +++ b/passwd-default @@ -1,10 +1,9 @@ admin:uaXjyn4Uw3qXo -sandbox_user:0kcQdq23ysbq. -anusha:pMfrTZVXm5Ix. admin2:IFWZaH87O7ZDg admin3:b6TA/1MC7CD96 +sandbox_user:0kcQdq23ysbq. +sandbox_user2:Zg8jCIxXK8/Sc +sandbox_user3:qhc7aFzy.y5vU sandbox_manager:ej4TeOgLu4GQ6 sandbox_manager2:blUNnoUzOfRNM sandbox_manager3:i/VlXSPqIgnwQ -sandbox_user2:Zg8jCIxXK8/Sc -sandbox_user3:qhc7aFzy.y5vU diff --git a/development-jenkins.ini b/production-jenkins.ini similarity index 67% rename from development-jenkins.ini rename to production-jenkins.ini index dd0421b..66c8267 100644 --- a/development-jenkins.ini +++ b/production-jenkins.ini @@ -1,3 +1,23 @@ +# Copyright (c) 2012 University of Oxford +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # rdfdatabank - Pylons development environment configuration # @@ -32,13 +52,14 @@ who.config_file = %(here)s/who.ini who.log_level = info who.log_file = /var/log/databank/who.log #who.log_file = %(here)s/logs/who.log -#who.log_file = stdout redis.host = localhost granary.store = %(here)s/silos granary.uri_root = http://dataflow-jenkins.bodleian.ox.ac.uk/ -#granary.uri_root = http://163.1.127.173/ + +#profile.log_filename = %(here)s/logs/profile.log +#profile.path = /__profile__ auth.file = %(here)s/passwd auth.info = %(here)s/rdfdatabank/config/users.py @@ -49,14 +70,18 @@ doi.count = %(here)s/rdfdatabank/config/doi_count broadcast.to = redis broadcast.queue = silochanges -solr.host = http://localhost:8983/solr +metadata.embargoed = False + +solr.host = http://localhost:8080/solr naming_rule = [^0-9a-zA-Z_\-\:] formats_served = text/html,text/xhtml,text/plain,application/json,application/rdf+xml,text/xml,text/rdf+n3,application/x-turtle,text/rdf+ntriples,text/rdf+nt publisher = Bodleian Libraries, University of Oxford rights = http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4b224c -license = +license = CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode #license = Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. See http://creativecommons.org/licenses/by-nc-sa/3.0/ +api.version = 0.2 + # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings # here: @@ -68,7 +93,6 @@ license = # execute malicious code after an exception is raised. #set debug = false - # Logging configuration [loggers] keys = root, routes, rdfdatabank @@ -90,7 +114,7 @@ qualname = routes.middleware # "level = DEBUG" logs the route matched and routing variables. [logger_rdfdatabank] -level = DEBUG +level = INFO handlers = logfile qualname = rdfdatabank diff --git a/production.ini b/production.ini index a7c0da6..03f892f 100644 --- a/production.ini +++ b/production.ini @@ -1,3 +1,23 @@ +# Copyright (c) 2012 University of Oxford +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # rdfdatabank - Pylons development environment configuration # @@ -6,7 +26,7 @@ [DEFAULT] debug = false # Uncomment and replace with the address which should receive any error reports -email_to = anusha.ranganathan@bodleian.ox.ac.uk +email_to = you@yourdomain.com smtp_server = localhost error_email_from = paste@databank @@ -24,25 +44,24 @@ use = egg:rdfdatabank full_stack = true static_files = true -cache_dir = %(here)s/data +cache_dir = /var/cache/databank beaker.session.key = rdfdatabank beaker.session.secret = somesecret -who.config_file = %(here)s/who.ini +who.config_file = /var/lib/databank/who.ini who.log_level = info who.log_file = /var/log/databank/who.log -#who.log_file = %(here)s/logs/who.log redis.host = localhost granary.store = /silos -granary.uri_root = http://databank.ora.ox.ac.uk/ +granary.uri_root = http://databank/ -auth.file = %(here)s/passwd -auth.info = %(here)s/rdfdatabank/config/users.py +auth.file = /var/lib/databank/passwd +auth.info = /var/lib/databank/rdfdatabank/config/users.py -doi.config = %(here)s/rdfdatabank/config/doi_config.py -doi.count = %(here)s/rdfdatabank/config/doi_count +doi.config = /var/lib/databank/rdfdatabank/config/doi_config.py +doi.count = /var/lib/databank/rdfdatabank/config/doi_count broadcast.to = redis broadcast.queue = silochanges @@ -57,6 +76,8 @@ rights = http://ora.ouls.ox.ac.uk/objects/uuid%3A1d00eebb-8fed-46ad-8e38-45dbdb4 license = CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdomain/zero/1.0/legalcode #license = Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported License. See http://creativecommons.org/licenses/by-nc-sa/3.0/ +api.version = 0.2 + # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings # here: @@ -68,7 +89,6 @@ license = CC0 1.0 Universal (CC0 1.0). See http://creativecommons.org/publicdoma # execute malicious code after an exception is raised. #set debug = false - # Logging configuration [loggers] keys = root, routes, rdfdatabank @@ -105,7 +125,6 @@ class = FileHandler level = INFO formatter = generic args = ('/var/log/databank/databank.log', 'w') -#args = ('%(here)s/logs/databank.log', 'w') [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s diff --git a/rdfdatabank.egg-info/PKG-INFO b/rdfdatabank.egg-info/PKG-INFO index b1eff68..eaa2bed 100644 --- a/rdfdatabank.egg-info/PKG-INFO +++ b/rdfdatabank.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: rdfdatabank -Version: 0.1dev -Summary: UNKNOWN -Home-page: UNKNOWN -Author: UNKNOWN -Author-email: UNKNOWN -License: UNKNOWN +Version: 0.2 +Summary: RDF-enhanced, pairtree-backed storage API +Home-page: https://github.com/dataflow/RDFDatabank +Author: Anusha Ranganathan, Ben O'Steen +Author-email: anusha.ranganathan@ieee.org, bosteen@gmail.com +License: MIT License Description: UNKNOWN Platform: UNKNOWN diff --git a/rdfdatabank.egg-info/SOURCES.txt b/rdfdatabank.egg-info/SOURCES.txt index 41a75aa..48c1f04 100644 --- a/rdfdatabank.egg-info/SOURCES.txt +++ b/rdfdatabank.egg-info/SOURCES.txt @@ -16,18 +16,160 @@ rdfdatabank/config/__init__.py rdfdatabank/config/deployment.ini_tmpl rdfdatabank/config/environment.py rdfdatabank/config/middleware.py +rdfdatabank/config/namespaces.py rdfdatabank/config/routing.py +rdfdatabank/config/users-default.py rdfdatabank/controllers/__init__.py +rdfdatabank/controllers/about.py +rdfdatabank/controllers/account.py +rdfdatabank/controllers/admin.py +rdfdatabank/controllers/api.py +rdfdatabank/controllers/datasets.py +rdfdatabank/controllers/doi.py rdfdatabank/controllers/error.py +rdfdatabank/controllers/home.py +rdfdatabank/controllers/items.py +rdfdatabank/controllers/keywords.py +rdfdatabank/controllers/search.py +rdfdatabank/controllers/searching.py +rdfdatabank/controllers/silos.py +rdfdatabank/controllers/states.py +rdfdatabank/controllers/users.py +rdfdatabank/lib/HTTP_request.py rdfdatabank/lib/__init__.py rdfdatabank/lib/app_globals.py +rdfdatabank/lib/auth.py rdfdatabank/lib/base.py +rdfdatabank/lib/broadcast.py +rdfdatabank/lib/conneg.py +rdfdatabank/lib/doi_helper.py +rdfdatabank/lib/doi_schema.py +rdfdatabank/lib/file_unpack.py rdfdatabank/lib/helpers.py +rdfdatabank/lib/htpasswd.py +rdfdatabank/lib/ident_md.py +rdfdatabank/lib/search_term.py +rdfdatabank/lib/short_pid.py +rdfdatabank/lib/utils.py rdfdatabank/model/__init__.py -rdfdatabank/public/bg.png -rdfdatabank/public/favicon.ico -rdfdatabank/public/index.html -rdfdatabank/public/pylons-logo.gif +rdfdatabank/public/static/databank_logo.png +rdfdatabank/public/static/databank_logo_generic.png +rdfdatabank/public/static/jquery.js +rdfdatabank/public/static/style.css +rdfdatabank/public/static/js/html5.js +rdfdatabank/public/static/json_data/DatasetStateInfo-dataset1-version0.txt +rdfdatabank/public/static/json_data/DatasetStateInfo-dataset1-version1.txt +rdfdatabank/public/static/json_data/DatasetStateInfo-dataset2.txt +rdfdatabank/public/static/json_data/SiloStateInfo.txt +rdfdatabank/public/static/json_data/adminInformation.txt +rdfdatabank/public/static/json_data/adminInformationForSilo.txt +rdfdatabank/public/static/json_data/datasetInformation-version0.txt +rdfdatabank/public/static/json_data/datasetInformation-version1.txt +rdfdatabank/public/static/json_data/datasetInformation.txt +rdfdatabank/public/static/json_data/datasetSubdirInformation-version1.txt +rdfdatabank/public/static/json_data/datasetSubdirInformation-version3.txt +rdfdatabank/public/static/json_data/datasetSubdirInformation.txt +rdfdatabank/public/static/json_data/datasetsInSiloInformation.txt +rdfdatabank/public/static/json_data/itemInformationForDataset-old.txt +rdfdatabank/public/static/json_data/itemInformationForDataset.txt +rdfdatabank/public/static/json_data/itemInformationForZipFileinDataset.txt +rdfdatabank/public/static/json_data/siloInformation.txt +rdfdatabank/public/static/json_data/silos.txt +rdfdatabank/public/static/styles/basic.css +rdfdatabank/public/static/styles/chimpanzee.css +rdfdatabank/public/static/styles/ie.css +rdfdatabank/public/static/styles/marmoset.css +rdfdatabank/public/static/styles/print.css +rdfdatabank/public/static/styles/reset.css +rdfdatabank/public/static/styles/silverback.css +rdfdatabank/public/static/styles/squirrelMonkey.css +rdfdatabank/public/static/styles/images/blkdiamond.gif +rdfdatabank/public/static/styles/images/blksquare.gif +rdfdatabank/public/static/styles/images/csv.png +rdfdatabank/public/static/styles/images/delete-icon-24.png +rdfdatabank/public/static/styles/images/down_arrow.png +rdfdatabank/public/static/styles/images/down_arrow_black.png +rdfdatabank/public/static/styles/images/down_arrow_blue.png +rdfdatabank/public/static/styles/images/file-add-icon-24.png +rdfdatabank/public/static/styles/images/file-new-icon-24.png +rdfdatabank/public/static/styles/images/fminus.png +rdfdatabank/public/static/styles/images/fplus.png +rdfdatabank/public/static/styles/images/go-up-icon-24.png +rdfdatabank/public/static/styles/images/help-icon-16.png +rdfdatabank/public/static/styles/images/help-icon-24.png +rdfdatabank/public/static/styles/images/info-icon-16.png +rdfdatabank/public/static/styles/images/json.png +rdfdatabank/public/static/styles/images/link.png +rdfdatabank/public/static/styles/images/page-edit-icon-24.png +rdfdatabank/public/static/styles/images/state-icon-24.png +rdfdatabank/public/static/styles/images/unzip-icon-24.png +rdfdatabank/public/static/styles/images/unzip-icon-32.png +rdfdatabank/public/static/styles/images/up_arrow.png +rdfdatabank/public/static/styles/images/up_arrow_black.png +rdfdatabank/public/static/styles/images/up_arrow_blue.png +rdfdatabank/public/static/styles/images/icons/breadcrumb-arrow.png +rdfdatabank/templates/about.html +rdfdatabank/templates/admin_api.html +rdfdatabank/templates/admin_siloview.html +rdfdatabank/templates/admin_user.html +rdfdatabank/templates/admin_users.html +rdfdatabank/templates/alter_silo.html +rdfdatabank/templates/api.html +rdfdatabank/templates/atom_results.html +rdfdatabank/templates/base.html +rdfdatabank/templates/create_doi.html +rdfdatabank/templates/create_new_item.html +rdfdatabank/templates/create_new_silo.html +rdfdatabank/templates/datasets_api.html +rdfdatabank/templates/datasetview.html +rdfdatabank/templates/datasetview_version.html +rdfdatabank/templates/delete_item.html +rdfdatabank/templates/doiview.html +rdfdatabank/templates/embargo_form.html +rdfdatabank/templates/file_upload.html +rdfdatabank/templates/files_unpack.html +rdfdatabank/templates/files_unpack2.html +rdfdatabank/templates/footer.html +rdfdatabank/templates/header.html +rdfdatabank/templates/home.html +rdfdatabank/templates/item_file_upload.html +rdfdatabank/templates/items_api.html +rdfdatabank/templates/itemview.html +rdfdatabank/templates/itemview_version.html +rdfdatabank/templates/keywords.html +rdfdatabank/templates/list_of_datasets.html +rdfdatabank/templates/list_of_silos.html +rdfdatabank/templates/list_of_zipfiles.html +rdfdatabank/templates/login.html +rdfdatabank/templates/logout.html +rdfdatabank/templates/part_list.html +rdfdatabank/templates/part_list_display.html +rdfdatabank/templates/raw_search.html +rdfdatabank/templates/rdf_manifest.html +rdfdatabank/templates/rdf_manifest_form.html +rdfdatabank/templates/readme_section.html +rdfdatabank/templates/register_new_user.html +rdfdatabank/templates/search.html +rdfdatabank/templates/search_advanced.html +rdfdatabank/templates/search_form.html +rdfdatabank/templates/search_response_display.html +rdfdatabank/templates/searching.html +rdfdatabank/templates/silo_admin.html +rdfdatabank/templates/silo_metadata.html +rdfdatabank/templates/silos_api.html +rdfdatabank/templates/siloview.html +rdfdatabank/templates/states_api.html +rdfdatabank/templates/update_user.html +rdfdatabank/templates/zipfilesubitemview.html +rdfdatabank/templates/zipfileview.html +rdfdatabank/tests/RDFDatabankConfig-Jenkins.py +rdfdatabank/tests/RDFDatabankConfig.py +rdfdatabank/tests/TestAdmin.py +rdfdatabank/tests/TestSubmission.py +rdfdatabank/tests/TestSubmission_load.py +rdfdatabank/tests/TestSubmission_submitter.py rdfdatabank/tests/__init__.py -rdfdatabank/tests/test_models.py -rdfdatabank/tests/functional/__init__.py \ No newline at end of file +rdfdatabank/tests/pylons_init.py +rdfdatabank/tests/testlib/SparqlQueryTestCase.py +rdfdatabank/tests/testlib/TestUtils.py +rdfdatabank/tests/testlib/__init__.py \ No newline at end of file diff --git a/rdfdatabank.egg-info/requires.txt b/rdfdatabank.egg-info/requires.txt index a6822b7..7e42a3a 100644 --- a/rdfdatabank.egg-info/requires.txt +++ b/rdfdatabank.egg-info/requires.txt @@ -1 +1,10 @@ -Pylons>=0.9.7 \ No newline at end of file +Pylons>=0.9.7 +pairtree>=0.5.6-T +recordsilo +redis +repoze.who>=2.0a4 +repoze.who_friendlyform +solrpy +lxml +rdflib==2.4.2 +python-dateutil>=1.4.1 diff --git a/rdfdatabank/config/environment.py b/rdfdatabank/config/environment.py index f7c75cc..d2d1528 100644 --- a/rdfdatabank/config/environment.py +++ b/rdfdatabank/config/environment.py @@ -1,5 +1,29 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """Pylons environment configuration""" + import os from mako.lookup import TemplateLookup diff --git a/rdfdatabank/config/middleware.py b/rdfdatabank/config/middleware.py index d65d79e..ae99e76 100644 --- a/rdfdatabank/config/middleware.py +++ b/rdfdatabank/config/middleware.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """Pylons middleware initialization""" #from paste import httpexceptions diff --git a/rdfdatabank/config/namespaces.py b/rdfdatabank/config/namespaces.py new file mode 100644 index 0000000..cbfd32f --- /dev/null +++ b/rdfdatabank/config/namespaces.py @@ -0,0 +1,23 @@ +from rdflib import Namespace + +NAMESPACES = {} +NAMESPACES['rdf'] = Namespace(u'http://www.w3.org/1999/02/22-rdf-syntax-ns#') +NAMESPACES['rdfs'] = Namespace(u'http://www.w3.org/2000/01/rdf-schema#') +NAMESPACES['dc'] = Namespace(u'http://purl.org/dc/elements/1.1/') +NAMESPACES['dcterms'] = Namespace(u'http://purl.org/dc/terms/') +NAMESPACES['foaf'] = Namespace(u'http://xmlns.com/foaf/0.1/') +NAMESPACES['oxds'] = Namespace(u'http://vocab.ox.ac.uk/dataset/schema#') +NAMESPACES['ore'] = Namespace(u'http://www.openarchives.org/ore/terms/') +NAMESPACES['bibo'] = Namespace(u'http://purl.org/ontology/bibo/') + +PREFIXES = {} +PREFIXES['http://www.w3.org/1999/02/22-rdf-syntax-ns#'] = 'rdf' +PREFIXES['http://www.w3.org/2000/01/rdf-schema#'] = 'rdfs' +PREFIXES['http://purl.org/dc/elements/1.1/'] = 'dc' +PREFIXES['http://purl.org/dc/terms/'] = 'dcterms' +PREFIXES['http://xmlns.com/foaf/0.1/'] = 'foaf' +PREFIXES['http://vocab.ox.ac.uk/dataset/schema#'] = 'oxds' +PREFIXES['http://www.openarchives.org/ore/terms/'] = 'ore' +PREFIXES['http://purl.org/ontology/bibo/'] = 'bibo' + + diff --git a/rdfdatabank/config/routing.py b/rdfdatabank/config/routing.py index dbf940d..55ab02a 100644 --- a/rdfdatabank/config/routing.py +++ b/rdfdatabank/config/routing.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """Routes configuration The more specific and detailed routes should be defined first so they @@ -21,14 +44,10 @@ def make_map(): # CUSTOM ROUTES HERE - #map.redirect("/", "/silos") map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently') - #Special controller to redirect datasets from databank.ouls to databank.ora - map.connect('/objects/{id}', controller='redirect', action='index') - - map.connect("/login", controller='account', action='login') + #Special controller to redirect datasets from databank.ouls to databank.ora map.connect('/objects/{id}', controller='redirect', action='index') map.connect("/login", controller='account', action='login') map.connect("/logout", controller='account', action='logout') map.connect("/welcome", controller='account', action='welcome') @@ -36,15 +55,18 @@ def make_map(): map.connect('/api', controller='api', action='index') map.connect('/api/{api_name}', controller='api', action='apiview') + map.connect('/keywords', controller='keywords', action='index') + map.connect('/about', controller='about', action='index') + map.connect('/searching', controller='searching', action='index') + map.connect('/admin', controller='admin', action='index') + map.connect('/silos', controller='silos', action='index') + map.connect('/{silo}', controller='silos', action='siloview') + map.connect('/{silo_name}/admin', controller='admin', action='archive') map.connect('/{silo_name}/register', controller='admin', action='register') - map.connect('/{silo_name}/users', controller='users', action='index') map.connect('/{silo_name}/users/{username}', controller='users', action='userview') - - map.connect('/silos', controller='silos', action='index') - map.connect('/{silo}', controller='silos', action='siloview') map.connect('/{silo}/datasets', controller='datasets', action='siloview') map.connect('/{silo}/datasets/{id}', controller='datasets', action='datasetview') diff --git a/rdfdatabank/config/users-default.py b/rdfdatabank/config/users-default.py index ce85c7c..2c25dac 100644 --- a/rdfdatabank/config/users-default.py +++ b/rdfdatabank/config/users-default.py @@ -1,2 +1,35 @@ #-*- coding: utf-8 -*- -_USERS = {'admin': {'owner': '*', 'first_name': 'Databank', 'last_name': 'Admin', 'role': 'admin'}, 'sandbox_user': {'owner': ['sandbox'], 'role': 'user', 'name': 'Sandbox user', 'description': 'User for silo Sandbox'}} +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +_USERS = { +'admin': {'owner': '*', 'first_name': 'Databank', 'last_name': 'Admin', 'role': 'admin', 'description': 'Admin for all silos'}, +'admin2': {'owner': ['sandbox'], 'first_name': 'Databank', 'last_name': 'Admin-2', 'role': 'admin', 'description': 'Admin for silo Sandbox'}, +'admin3': {'owner': ['sandbox2'], 'first_name': 'Databank', 'last_name': 'Admin-3', 'role': 'admin', 'description': 'Admin for silo Sandbox2'}, +'sandbox_user': {'owner': ['sandbox'], 'role': 'user', 'name': 'Sandbox user', 'description': 'User for silo Sandbox'}, +'sandbox_user2': {'owner': ['sandbox'], 'role': 'user', 'name': 'Sandbox user-2', 'description': 'User for silo Sandbox'}, +'sandbox_user3': {'owner': ['sandbox2'], 'role': 'user', 'name': 'Sandbox user-3', 'description': 'User for silo Sandbox2'}, +'sandbox_manager': {'owner': ['sandbox'], 'role': 'manager', 'name': 'Sandbox manager', 'description': 'Manager for silo Sandbox'}, +'sandbox_manager2': {'owner': ['sandbox'], 'role': 'manager', 'name': 'Sandbox manager-2', 'description': 'Manager for silo Sandbox'}, +'sandbox_manager3': {'owner': ['sandbox2'], 'role': 'manager', 'name': 'Sandbox manager-3', 'description': 'Manager for silo Sandbox2'} +} diff --git a/rdfdatabank/controllers/about.py b/rdfdatabank/controllers/about.py new file mode 100644 index 0000000..b40290f --- /dev/null +++ b/rdfdatabank/controllers/about.py @@ -0,0 +1,31 @@ +#-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import logging + +from rdfdatabank.lib.base import BaseController, render + +class AboutController(BaseController): + def index(self): + return render('/about.html') diff --git a/rdfdatabank/controllers/account.py b/rdfdatabank/controllers/account.py index 230efa4..063d4f2 100644 --- a/rdfdatabank/controllers/account.py +++ b/rdfdatabank/controllers/account.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from pylons import url from pylons import request, response, session, tmpl_context as c, url, app_globals as ag from rdfdatabank.lib.base import BaseController, render diff --git a/rdfdatabank/controllers/admin.py b/rdfdatabank/controllers/admin.py index 50f6d57..0f97a15 100644 --- a/rdfdatabank/controllers/admin.py +++ b/rdfdatabank/controllers/admin.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import simplejson from pylons import request, response, session, config, tmpl_context as c, url @@ -138,12 +161,12 @@ def archive(self, silo_name): #c.granary_list = ag.granary.silos c.silo_name = silo_name # Admin only - if not ident.get('role') == "admin": + if not ident.get('role') in ["admin", "manager"]: abort(403, "Do not have admin credentials") if ident.get('role') == "admin": c.roles = ["admin", "manager", "user"] else: - c.roles = ["admin", "manager"] + c.roles = ["manager", "user"] granary_list = ag.granary.silos silos = ag.authz(granary_list, ident) if not silo_name in silos: @@ -317,11 +340,17 @@ def archive(self, silo_name): # and then remove the silo todelete_silo = ag.granary.get_rdf_silo(silo_name) for item in todelete_silo.list_items(): - ag.b.deletion(silo_name, item, ident=ident['repoze.who.userid']) + try: + ag.b.deletion(silo_name, item, ident=ident['repoze.who.userid']) + except: + pass ag.granary.delete_silo(silo_name) + try: + ag.b.silo_deletion(silo_name, ident=ident['repoze.who.userid']) + except: + pass - ag.b.silo_deletion(silo_name, ident=ident['repoze.who.userid']) try: del ag.granary.state[silo_name] except: diff --git a/rdfdatabank/controllers/api.py b/rdfdatabank/controllers/api.py index 204d605..ab8a779 100644 --- a/rdfdatabank/controllers/api.py +++ b/rdfdatabank/controllers/api.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging from pylons import request, response, session, tmpl_context as c, url diff --git a/rdfdatabank/controllers/datasets.py b/rdfdatabank/controllers/datasets.py index 1b96b05..bbbff7d 100644 --- a/rdfdatabank/controllers/datasets.py +++ b/rdfdatabank/controllers/datasets.py @@ -1,4 +1,27 @@ #-*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import re, os, shutil, codecs import simplejson @@ -12,7 +35,8 @@ from pylons.decorators import rest from paste.fileapp import FileApp from rdfdatabank.lib.base import BaseController, render -from rdfdatabank.lib.utils import create_new, is_embargoed, get_readme_text, test_rdf, munge_manifest, serialisable_stat, allowable_id2, get_embargo_values, get_rdf_template +from rdfdatabank.lib.utils import create_new, get_readme_text, serialisable_stat, allowable_id2, natural_sort +from rdfdatabank.lib.utils import is_embargoed, test_rdf, munge_manifest, get_embargo_values, get_rdf_template, extract_metadata from rdfdatabank.lib.file_unpack import get_zipfiles_in_dataset from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse @@ -27,9 +51,6 @@ def siloview(self, silo): abort(404) c.silo_name = silo granary_list = ag.granary.silos - #f = open('/tmp/ds.log', 'a') - #f.write(str(granary_list)) - #f.close() ident = request.environ.get('repoze.who.identity') c.ident = ident @@ -118,7 +139,10 @@ def siloview(self, silo): item = create_new(c_silo, id, ident['repoze.who.userid'], **params) # Broadcast change as message - ag.b.creation(silo, id, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, ident=ident['repoze.who.userid']) + except: + pass # conneg return accept_list = None @@ -244,6 +268,10 @@ def datasetview(self, silo, id): c.embargos[id] = is_embargoed(c_silo, id) c.parts = item.list_parts(detailed=True) c.manifest_pretty = item.rdf_to_string(format="pretty-xml") + c.metadata = None + c.metadata = extract_metadata(item) + c.versions = item.manifest['versions'] + c.versions = natural_sort(c.versions) #c.manifest = item.rdf_to_string() c.manifest = get_rdf_template(item.uri, id) c.zipfiles = get_zipfiles_in_dataset(item) @@ -329,7 +357,10 @@ def datasetview(self, silo, id): item = create_new(c_silo, id, ident['repoze.who.userid'], **params) # Broadcast change as message - ag.b.creation(silo, id, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, ident=ident['repoze.who.userid']) + except: + pass # conneg return accept_list = None @@ -379,24 +410,35 @@ def datasetview(self, silo, id): e, e_d = get_embargo_values(embargoed=params['embargoed'], embargo_days_from_now=params['embargo_days_from_now']) else: e, e_d = get_embargo_values(embargoed=params['embargoed']) - item.metadata['embargoed_until'] = '' item.del_triple(item.uri, u"oxds:isEmbargoed") item.del_triple(item.uri, u"oxds:embargoedUntil") - ag.r.set("%s:%s:embargoed_until" % (c_silo.state['storage_dir'], id), ' ') + try: + ag.r.set("%s:%s:embargoed_until" % (c_silo.state['storage_dir'], id), ' ') + except: + pass if e: item.metadata['embargoed'] = True item.add_triple(item.uri, u"oxds:isEmbargoed", 'True') - ag.r.set("%s:%s:embargoed" % (c_silo.state['storage_dir'], id), True) + try: + ag.r.set("%s:%s:embargoed" % (c_silo.state['storage_dir'], id), True) + except: + pass if e_d: item.metadata['embargoed_until'] = e_d item.add_triple(item.uri, u"oxds:embargoedUntil", e_d) - ag.r.set("%s:%s:embargoed_until" % (c_silo.state['storage_dir'], id), e_d) + try: + ag.r.set("%s:%s:embargoed_until" % (c_silo.state['storage_dir'], id), e_d) + except: + pass else: item.metadata['embargoed'] = False item.add_triple(item.uri, u"oxds:isEmbargoed", 'False') - ag.r.set("%s:%s:embargoed" % (c_silo.state['storage_dir'], id), False) + try: + ag.r.set("%s:%s:embargoed" % (c_silo.state['storage_dir'], id), False) + except: + pass item.del_triple(item.uri, u"dcterms:modified") item.add_triple(item.uri, u"dcterms:modified", datetime.now()) @@ -496,13 +538,19 @@ def datasetview(self, silo, id): item.sync() if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "201 Created" response.status_int = 201 response.headers["Content-Location"] = url(controller="datasets", action="itemview", id=id, silo=silo, path=filename) response_message = "201 Created. Added file %s to item %s" % (filename, id) else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "204 Updated" response.status_int = 204 response_message = None @@ -589,13 +637,19 @@ def datasetview(self, silo, id): item.sync() if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "201 Created" response.status_int = 201 response.headers["Content-Location"] = url(controller="datasets", action="datasetview", id=id, silo=silo) response_message = "201 Created. Added file %s to item %s" % (filename, id) else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "204 Updated" response.status_int = 204 response_message = None @@ -642,7 +696,10 @@ def datasetview(self, silo, id): c_silo.del_item(id) # Broadcast deletion - ag.b.deletion(silo, id, ident=ident['repoze.who.userid']) + try: + ag.b.deletion(silo, id, ident=ident['repoze.who.userid']) + except: + pass response.content_type = "text/plain" response.status_int = 200 @@ -853,6 +910,8 @@ def itemview(self, silo, id, path): return fileserve_app(request.environ, self.start_response) elif item.isdir(path): #c.parts = item.list_parts(detailed=True) + c.versions = item.manifest['versions'] + c.versions = natural_sort(c.versions) c.parts = item.list_parts(path, detailed=True) c.readme_text = None if "README" in c.parts.keys(): @@ -940,13 +999,19 @@ def itemview(self, silo, id, path): item.sync() if code == 201: - ag.b.creation(silo, id, path, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, path, ident=ident['repoze.who.userid']) + except: + pass response.status = "201 Created" response.status_int = 201 response.headers["Content-Location"] = url(controller="datasets", action="itemview", id=id, silo=silo, path=path) response_message = "201 Created" else: - ag.b.change(silo, id, path, ident=ident['repoze.who.userid']) + try: + ag.b.change(silo, id, path, ident=ident['repoze.who.userid']) + except: + pass response.status = "204 Updated" response.status_int = 204 response_message = None @@ -1036,13 +1101,19 @@ def itemview(self, silo, id, path): item.sync() if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "201 Created" response.status_int = 201 response.headers["Content-Location"] = url(controller="datasets", action="itemview", id=id, silo=silo, path=path) response_message = "201 Created" else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + try: + ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) + except: + pass response.status = "204 Updated" response.status_int = 204 response_message = None @@ -1089,7 +1160,10 @@ def itemview(self, silo, id, path): item.del_triple(item.uri, u"oxds:currentVersion") item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) item.sync() - ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) + try: + ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) + except: + pass response.content_type = "text/plain" response.status_int = 200 response.status = "200 OK" @@ -1102,7 +1176,10 @@ def itemview(self, silo, id, path): item.del_triple(item.uri, u"dcterms:modified") item.add_triple(item.uri, u"dcterms:modified", datetime.now()) item.sync() - ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) + try: + ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) + except: + pass response.content_type = "text/plain" response.status_int = 200 response.status = "200 OK" diff --git a/rdfdatabank/controllers/datasets_log.py b/rdfdatabank/controllers/datasets_log.py deleted file mode 100644 index e116dde..0000000 --- a/rdfdatabank/controllers/datasets_log.py +++ /dev/null @@ -1,1237 +0,0 @@ -#-*- coding: utf-8 -*- -import logging -import re, os, shutil -import simplejson -from datetime import datetime, timedelta -import time - -from pylons import request, response, session, tmpl_context as c, url, app_globals as ag -from pylons.controllers.util import abort, redirect_to -from pylons.decorators import rest -from paste.fileapp import FileApp -from rdfdatabank.lib.base import BaseController, render -from rdfdatabank.lib.utils import create_new, is_embargoed, get_readme_text, test_rdf, munge_manifest, manifest_type, serialisable_stat, allowable_id2 -from rdfdatabank.lib.file_unpack import get_zipfiles_in_dataset -from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse - -JAILBREAK = re.compile("[\/]*\.\.[\/]*") - -log = logging.getLogger(__name__) - -class DatasetsController(BaseController): - @rest.restrict('GET', 'POST') - def siloview(self, silo): - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - - c.silo_name = silo - c_silo = ag.granary.get_rdf_silo(silo) - - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - hr = "-"*80 + '\n' - f.write(hr) - f.write("%s :: datasets - siloview : silo=%s\n"%(str(datetime.now()), silo)) - - http_method = request.environ['REQUEST_METHOD'] - - f.write("Http method = %s\n"%http_method) - f.close() - - if http_method == "GET": - c.embargos = {} - for item in c_silo.list_items(): - c.embargos[item] = is_embargoed(c_silo, item) - #c.items = c_silo.list_items() - c.items = c.embargos.keys() - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render('/siloview.html') - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return simplejson.dumps(c.embargos) - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops nothing satisfies - return text/html - return render('/siloview.html') - elif http_method == "POST": - params = request.POST - if params.has_key("id"): - if c_silo.exists(params['id']): - response.content_type = "text/plain" - response.status_int = 409 - response.status = "409 Conflict: Dataset Already Exists" - return "Dataset Already Exists" - else: - # Supported params: - # id, title, embargoed, embargoed_until, embargo_days_from_now - id = params['id'] - if not allowable_id2(id): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Dataset name can contain only the following characters - %s and has to be more than 1 character"%ag.naming_rule - del params['id'] - item = create_new(c_silo, id, ident['repoze.who.userid'], **params) - - # Broadcast change as message - ag.b.creation(silo, id, ident=ident['repoze.who.userid']) - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - redirect_to(controller="datasets", action="datasetview", silo=silo, id=id) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=id) - return "201 Created" - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - # Whoops - nothing satisfies - return text/plain - response.content_type = "text/plain" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=id) - response.status = "201 Created" - return "201 Created" - - @rest.restrict('GET', 'POST', 'DELETE') - def datasetview(self, silo, id): - # Check to see if embargo is on: - c.silo_name = silo - c.id = id - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - hr = "-"*80 + '\n' - f.write(hr) - f.write("%s :: datasets - datasetview : silo=%s, id=%s\n"%(str(datetime.now()), silo, id)) - - http_method = request.environ['REQUEST_METHOD'] - f.write("Http method = %s\n"%http_method) - - # -- Step 1 ----------------------------- - tic1 = time.mktime(time.gmtime()) - c_silo = ag.granary.get_rdf_silo(silo) - toc = time.mktime(time.gmtime()) - f.write("1. Get rdf silo: %d\n"%(toc-tic1)) - f.close() - - c.editor = False - c.version = None - - # -- Step 2 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - if not (http_method == "GET"): - #identity management of item - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - c.editor = silo in silos - elif http_method == "GET": - if not c_silo.exists(id): - abort(404) - item = c_silo.get_item(id) - embargoed = False - if item.metadata.get('embargoed') not in ["false", 0, False]: - embargoed = True - granary_list = ag.granary.silos - if embargoed: - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - c.editor = silo in silos - ident = request.environ.get('repoze.who.identity') - c.ident = ident - if ident: - silos = ag.authz(granary_list, ident) - c.editor = silo in silos - toc = time.mktime(time.gmtime()) - f.write("2. Authentication: %d\n"%(toc-tic)) - f.close() - - # Method determination - if http_method == "GET": - # -- Step 3 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.embargos = {} - c.embargos[id] = is_embargoed(c_silo, id) - toc = time.mktime(time.gmtime()) - f.write("3. Embargo information: %d\n"%(toc-tic)) - f.close() - - # -- Step 4 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.parts = item.list_parts(detailed=True) - toc = time.mktime(time.gmtime()) - f.write("4. Detailed file stats: %d\n"%(toc-tic)) - f.close() - - # -- Step 5 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.manifest_pretty = item.rdf_to_string(format="pretty-xml") - toc = time.mktime(time.gmtime()) - f.write("5. Pretty manifest: %d\n"%(toc-tic)) - f.close() - - # -- Step 6 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.manifest = item.rdf_to_string() - toc = time.mktime(time.gmtime()) - f.write("6. Manifest: %d\n"%(toc-tic)) - f.close() - - # -- Step 7 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.zipfiles = get_zipfiles_in_dataset(item) - toc = time.mktime(time.gmtime()) - f.write("7. List of zipfiles: %d\n"%(toc-tic)) - f.close() - - # -- Step 8 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - c.readme_text = None - #if item.isfile("README"): - if "README" in c.parts.keys(): - c.readme_text = get_readme_text(item) - #if item.manifest: - # state = item.manifest.state - toc = time.mktime(time.gmtime()) - f.write("8. Readme Text: %d\n"%(toc-tic)) - f.close() - - # View options - options = request.GET - if "view" in options: - c.view = options['view'] - elif c.editor: - c.view = 'editor' - else: - c.view = 'user' - - # conneg: - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - # ------------------------------- - while(mimetype): - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Trying MIME type %s\n"%mimetype) - f.close() - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/html, text/xhtml\n") - f.close() - return render('/datasetview.html') - elif str(mimetype).lower() in ["text/plain", "application/json"]: - # -- Step 9 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - tic = time.mktime(time.gmtime()) - response.content_type = 'application/json; charset="UTF-8"' - returndata = {} - returndata['embargos'] = c.embargos - returndata['view'] = c.view - returndata['editor'] = c.editor - returndata['parts'] = {} - for part in c.parts: - returndata['parts'][part] = serialisable_stat(c.parts[part]) - returndata['readme_text'] = c.readme_text - returndata['manifest_pretty'] = c.manifest_pretty - returndata['manifest'] = c.manifest - returndata['zipfiles'] = c.zipfiles - #items['state'] = state - toc = time.mktime(time.gmtime()) - f.write("9. Accumulate JSON data: %d\n"%(toc-tic)) - #f.write("Data returned : \n %s\n\n"%str(returndata)) - f.write("--END-- Data returned for application/json, text/plain\n") - f.close() - response.status_int = 200 - response.status = "200 OK" - return simplejson.dumps(returndata) - elif str(mimetype).lower() in ["application/rdf+xml", "text/xml"]: - # -- Step 9 ----------------------------- - response.status_int = 200 - response.status = "200 OK" - response.content_type = 'application/rdf+xml; charset="UTF-8"' - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Data returned for application/rdf+xml, text/xml\n") - #f.write(c.manifest_pretty + '\n') - f.close() - return c.manifest_pretty - elif str(mimetype).lower() == "text/rdf+n3": - response.content_type = 'text/rdf+n3; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/rdf+n3\n") - f.close() - return item.rdf_to_string(format="n3") - elif str(mimetype).lower() == "application/x-turtle": - response.content_type = 'application/x-turtle; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for application/x-turtle\n") - f.close() - return item.rdf_to_string(format="turtle") - elif str(mimetype).lower() in ["text/rdf+ntriples", "text/rdf+nt"]: - response.content_type = 'text/rdf+ntriples; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/rdf+ntriples, text/rdf+nt\n") - f.close() - return item.rdf_to_string(format="nt") - # Whoops - nothing satisfies - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing staisfies - default to text/html - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for Default (text/html)\n") - f.close() - return render('/datasetview.html') - elif http_method == "POST" and c.editor: - params = request.POST - if not c_silo.exists(id): - if not allowable_id2(id): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Dataset name can contain only the following characters - %s and has to be more than 1 character"%ag.naming_rule - if 'id' in params.keys(): - del params['id'] - item = create_new(c_silo, id, ident['repoze.who.userid'], **params) - - # Broadcast change as message - ag.b.creation(silo, id, ident=ident['repoze.who.userid']) - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Trying MIME type %s\n"%mimetype) - f.close() - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/html, text/xhtml\n") - f.close() - redirect_to(controller="datasets", action="datasetview", silo=silo, id=id) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=id) - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/plain, application/json\n") - f.close() - return "201 Created" - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - # Whoops - nothing satisfies - return text/plain - response.content_type = "text/plain" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=id) - response.status = "201 Created" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for default (text/plain)\n") - f.close() - return "201 Created" - elif params.has_key('embargo_change'): - item = c_silo.get_item(id) - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - if params.has_key('embargoed'): - if params.has_key('embargoed_until') and params['embargoed_until']: - embargoed_until_date = params['embargoed_until'] - elif params.has_key('embargo_days_from_now') and params['embargo_days_from_now']: - embargoed_until_date = (datetime.now() + timedelta(days=params['embargo_days_from_now'])).isoformat() - else: - embargoed_until_date = (datetime.now() + timedelta(days=365*70)).isoformat() - item.metadata['embargoed'] = True - item.metadata['embargoed_until'] = embargoed_until_date - item.del_triple(item.uri, u"oxds:isEmbargoed") - item.del_triple(item.uri, u"oxds:embargoedUntil") - item.add_triple(item.uri, u"oxds:isEmbargoed", 'True') - item.add_triple(item.uri, u"oxds:embargoedUntil", embargoed_until_date) - else: - #if is_embargoed(c_silo, id)[0] == True: - item.metadata['embargoed'] = False - item.metadata['embargoed_until'] = '' - item.del_triple(item.uri, u"oxds:isEmbargoed") - item.del_triple(item.uri, u"oxds:embargoedUntil") - item.add_triple(item.uri, u"oxds:isEmbargoed", 'False') - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - item.sync() - e, e_d = is_embargoed(c_silo, id, refresh=True) - - # Broadcast change as message - ag.b.embargo_change(silo, id, item.metadata['embargoed'], item.metadata['embargoed_until'], ident=ident['repoze.who.userid']) - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Trying MIME type %s\n"%mimetype) - f.close() - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/html, text/xhtml\n") - f.close() - redirect_to(controller="datasets", action="datasetview", id=id, silo=silo) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - response.status_int = 204 - response.status = "204 Updated" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/plain, application/json\n") - f.close() - return - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text / plain - response.content_type = "text/plain" - response.status_int = 204 - response.status = "204 Updated" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for default (text/plain)\n") - f.close() - return - elif params.has_key('file'): - # File upload by a not-too-savvy method - Service-orientated fallback: - # Assume file upload to 'filename' - params = request.POST - item = c_silo.get_item(id) - filename = params.get('filename') - if not filename: - filename = params['file'].filename - upload = params.get('file') - if JAILBREAK.search(filename) != None: - abort(400, "'..' cannot be used in the path or as a filename") - target_path = filename - - if item.isfile(target_path): - code = 204 - elif item.isdir(target_path): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Cannot POST a file on to an existing directory" - else: - code = 201 - - if filename == "manifest.rdf": - #Copy the uploaded file to a tmp area - mani_file = os.path.join('/tmp', filename.lstrip(os.sep)) - mani_file_obj = open(mani_file, 'w') - shutil.copyfileobj(upload.file, mani_file_obj) - upload.file.close() - mani_file_obj.close() - #test rdf file - mani_file_obj = open(mani_file, 'r') - manifest_str = mani_file_obj.read() - mani_file_obj.close() - if not test_rdf(manifest_str): - response.status_int = 400 - return "Bad manifest file" - #munge rdf - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - a = item.get_rdf_manifest() - b = a.to_string() - mtype = manifest_type(b) - if not mtype: - mtype = 'http://vocab.ox.ac.uk/dataset/schema#Grouping' - munge_manifest(manifest_str, item, manifest_type=mtype) - else: - if code == 204: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf', filename]) - else: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.put_stream(target_path, upload.file) - upload.file.close() - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - - if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "201 Created" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", id=id, silo=silo) - response_message = "201 Created. Added file %s to item %s" % (filename, id) - else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "204 Updated" - response.status_int = 204 - response_message = None - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Trying MIME type %s\n"%mimetype) - f.close() - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/html, text/xhtml\n") - f.close() - redirect_to(controller="datasets", action="datasetview", id=id, silo=silo) - elif str(mimetype).lower() in ["text/plain"]: - response.content_type = "text/plain" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/plain, application/json\n") - f.close() - return response_message - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text / plain - response.content_type = "text/plain" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for default (text/plain)\n") - f.close() - return response_message - elif params.has_key('text'): - # Text upload convenience service - params = request.POST - item = c_silo.get_item(id) - filename = params.get('filename') - if not filename: - abort(406, "Must supply a filename") - - if JAILBREAK.search(filename) != None: - abort(400, "'..' cannot be used in the path or as a filename") - target_path = filename - - if item.isfile(target_path): - code = 204 - elif item.isdir(target_path): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 forbidden" - return "Cannot POST a file on to an existing directory" - else: - code = 201 - - if filename == "manifest.rdf": - # valid to make sure it's valid RDF - # Otherwise this dataset will not be accessible - text = params['text'] - if not test_rdf(text): - abort(406, "Not able to parse RDF/XML") - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - a = item.get_rdf_manifest() - b = a.to_string() - mtype = manifest_type(b) - if not mtype: - mtype = 'http://vocab.ox.ac.uk/dataset/schema#Grouping' - munge_manifest(text, item, manifest_type=mtype) - else: - if code == 204: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf', filename]) - else: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.put_stream(target_path, params['text'].encode("utf-8")) - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - - if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "201 Created" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", id=id, silo=silo) - response_message = "201 Created. Added file %s to item %s" % (filename, id) - else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "204 Updated" - response.status_int = 204 - response_message = None - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Trying MIME type %s\n"%mimetype) - f.close() - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/html, text/xhtml\n") - f.close() - redirect_to(controller="datasets", action="datasetview", id=id, silo=silo) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for text/plain, application/json\n") - f.close() - return response_message - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text / plain - response.content_type = "text/plain" - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("--END-- Data returned for default (text/plain)\n") - f.close() - return response_message - else: - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "403 Forbidden" - elif http_method == "DELETE" and c.editor: - if c_silo.exists(id): - c_silo.del_item(id) - - # Broadcast deletion - ag.b.deletion(silo, id, ident=ident['repoze.who.userid']) - - response.content_type = "text/plain" - response.status_int = 200 - response.status = "200 OK" - return "{'ok':'true'}" # required for the JQuery magic delete to succede. - else: - abort(404) - - @rest.restrict('GET') - def datasetview_vnum(self, silo, id, vnum): - c.silo_name = silo - c.id = id - c_silo = ag.granary.get_rdf_silo(silo) - - if not c_silo.exists(id): - abort(404) - - item = c_silo.get_item(id) - vnum = str(vnum) - if not vnum in item.manifest['versions']: - abort(404) - #Set the item's version cursor - item.set_version_cursor(vnum) - c.version = vnum - - embargoed = False - c.editor = False - if item.metadata.get('embargoed') not in ["false", 0, False]: - embargoed = True - if embargoed: - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorized") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - c.editor = silo in silos - elif request.environ.get('repoze.who.identity'): - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - c.editor = silo in silos - - # Check to see if embargo is on: - c.embargos = {} - c.embargos[id] = is_embargoed(c_silo, id) - c.readme_text = None - c.parts = item.list_parts(detailed=True) - c.manifest_pretty = item.rdf_to_string(format="pretty-xml") - if "README" in c.parts.keys(): - c.readme_text = get_readme_text(item) - - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - mimetype = accept_list.pop(0) - - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render('/datasetview_version.html') - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - returndata = {} - returndata['embargos'] = c.embargos - returndata['view'] = c.view - returndata['editor'] = c.editor - returndata['parts'] = {} - for part in c.parts: - returndata['parts'][part] = serialisable_stat(c.parts[part]) - returndata['readme_text'] = c.readme_text - returndata['manifest_pretty'] = c.manifest_pretty - return simplejson.dumps(items) - elif str(mimetype).lower() in ["application/rdf+xml", "text/xml"]: - response.content_type = 'application/rdf+xml; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return c.manifest_pretty - elif str(mimetype).lower() == "text/rdf+n3": - response.content_type = 'text/rdf+n3; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return item.rdf_to_string(format="n3") - elif str(mimetype).lower() == "application/x-turtle": - response.content_type = 'application/x-turtle; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return item.rdf_to_string(format="turtle") - elif str(mimetype).lower() in ["text/rdf+ntriples", "text/rdf+nt"]: - response.content_type = 'text/rdf+ntriples; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return item.rdf_to_string(format="nt") - # Whoops - nothing satisfies - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing staisfies - default to text/html - return render('/datasetview_version.html') - - @rest.restrict('GET', 'POST', 'PUT', 'DELETE') - def itemview(self, silo, id, path): - # Check to see if embargo is on: - c.silo_name = silo - c.id = id - c.version = None - c.path = path - c_silo = ag.granary.get_rdf_silo(silo) - - if not c_silo.exists(id): - # dataset doesn't exist yet... - # DECISION FOR POST / PUT : Auto-instantiate dataset and then put file there? - # or error out with perhaps a 404? - # Going with error out... - abort(404) - - embargoed = False - item = c_silo.get_item(id) - if item.metadata.get('embargoed') not in ["false", 0, False]: - embargoed = True - - http_method = request.environ['REQUEST_METHOD'] - - editor = False - granary_list = ag.granary.silos - if not (http_method == "GET"): - #identity management if item - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - editor = silo in silos - elif embargoed: - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - editor = silo in silos - ident = request.environ.get('repoze.who.identity') - c.ident = ident - if ident: - silos = ag.authz(granary_list, ident) - editor = silo in silos - - if http_method == "GET": - if item.isfile(path): - fileserve_app = FileApp(item.to_dirpath(path)) - return fileserve_app(request.environ, self.start_response) - elif item.isdir(path): - #c.parts = item.list_parts(detailed=True) - c.parts = item.list_parts(path, detailed=True) - c.readme_text = None - if "README" in c.parts.keys(): - c.readme_text = get_readme_text(item, "%s/README" % path) - - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render("/itemview.html") - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - returndata = {} - returndata['parts'] = {} - for part in c.parts: - returndata['parts'][part] = serialisable_stat(c.parts[part]) - returndata['readme_text'] = c.readme_text - return simplejson.dumps(returndata) - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text/html - return render("/itemview.html") - else: - abort(404) - elif http_method == "PUT" and editor: - # Pylons loads the request body into request.body... - # This is not going to work for large files... ah well - # POST will handle large files as they are pushed to disc, - # but this won't - content = request.body - item = c_silo.get_item(id) - - if JAILBREAK.search(path) != None: - abort(400, "'..' cannot be used in the path") - - if item.isfile(path): - code = 204 - elif item.isdir(path): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Cannot PUT a file on to an existing directory" - else: - code = 201 - - #Check if path is manifest.rdf - If, yes Munge - if "manifest.rdf" in path: - #test content is valid rdf - if not test_rdf(content): - response.status_int = 400 - return "Bad manifest file" - #munge rdf - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - a = item.get_rdf_manifest() - b = a.to_string() - mtype = manifest_type(b) - if not mtype: - mtype = 'http://vocab.ox.ac.uk/dataset/schema#Grouping' - munge_manifest(content, item, manifest_type=mtype) - else: - if code == 204: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf', path]) - else: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.put_stream(path, content) - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - - if code == 201: - ag.b.creation(silo, id, path, ident=ident['repoze.who.userid']) - response.status = "201 Created" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="itemview", id=id, silo=silo, path=path) - response_message = "201 Created" - else: - ag.b.change(silo, id, path, ident=ident['repoze.who.userid']) - response.status = "204 Updated" - response.status_int = 204 - response_message = None - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - redirect_to(controller="datasets", action="itemview", id=id, silo=silo, path=path) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - return response_message - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text / plain - response.content_type = "text/plain" - return response_message - elif http_method == "POST" and editor: - # POST... differences from PUT: - # path = filepath that this acts on, should be dir, or non-existant - # if path is a file, this will revert to PUT's functionality and - # overwrite the file, if there is a multipart file uploaded - # Expected params: filename, file (uploaded file) - params = request.POST - item = c_silo.get_item(id) - filename = params.get('filename') - upload = params.get('file') - if JAILBREAK.search(filename) != None: - abort(400, "'..' cannot be used in the path or as a filename") - target_path = path - if item.isdir(path) and filename: - target_path = os.path.join(path, filename) - - if item.isfile(target_path): - code = 204 - elif item.isdir(target_path): - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Cannot POST a file on to an existing directory" - else: - code = 201 - - if filename == "manifest.rdf": - #Copy the uploaded file to a tmp area - mani_file = os.path.join('/tmp', filename.lstrip(os.sep)) - mani_file_obj = open(mani_file, 'w') - shutil.copyfileobj(upload.file, mani_file_obj) - upload.file.close() - mani_file_obj.close() - #test rdf file - mani_file_obj = open(mani_file, 'r') - manifest_str = mani_file_obj.read() - mani_file_obj.close() - if not test_rdf(manifest_str): - response.status_int = 400 - return "Bad manifest file" - #munge rdf - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - a = item.get_rdf_manifest() - b = a.to_string() - mtype = manifest_type(b) - if not mtype: - mtype = 'http://vocab.ox.ac.uk/dataset/schema#Grouping' - munge_manifest(manifest_str, item, manifest_type=mtype) - else: - if code == 204: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf', filename]) - else: - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.put_stream(target_path, upload.file) - upload.file.close() - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - - if code == 201: - ag.b.creation(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "201 Created" - response.status_int = 201 - response.headers["Content-Location"] = url(controller="datasets", action="itemview", id=id, silo=silo, path=path) - response_message = "201 Created" - else: - ag.b.change(silo, id, target_path, ident=ident['repoze.who.userid']) - response.status = "204 Updated" - response.status_int = 204 - response_message = None - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_dataset.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - redirect_to(controller="datasets", action="itemview", id=id, silo=silo, path=path) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - return response_message - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text / plain - response.content_type = "text/plain" - return response_message - elif http_method == "DELETE" and editor: - item = c_silo.get_item(id) - if item.isfile(path): - if 'manifest.rdf' in path: - response.content_type = "text/plain" - response.status_int = 403 - response.status = "403 Forbidden" - return "Forbidden - Cannot delete the manifest" - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.del_stream(path) - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - item.sync() - ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) - response.content_type = "text/plain" - response.status_int = 200 - response.status = "200 OK" - return "{'ok':'true'}" # required for the JQuery magic delete to succede. - elif item.isdir(path): - parts = item.list_parts(path) - for part in parts: - if item.isdir(os.path.join(path, part)): - # TODO implement proper recursive delete, with RDF aggregation - # updating - abort(400, "Directory is not empty of directories") - item.increment_version_delta(clone_previous_version=True, copy_filenames=['manifest.rdf']) - item.del_triple(item.uri, u"oxds:currentVersion") - item.add_triple(item.uri, u"oxds:currentVersion", item.currentversion) - for part in parts: - item.del_stream(os.path.join(path, part)) - ag.b.deletion(silo, id, os.path.join(path, part), ident=ident['repoze.who.userid']) - item.del_stream(path) - item.del_triple(item.uri, u"dcterms:modified") - item.add_triple(item.uri, u"dcterms:modified", datetime.now()) - item.sync() - ag.b.deletion(silo, id, path, ident=ident['repoze.who.userid']) - response.content_type = "text/plain" - response.status_int = 200 - response.status = "200 OK" - return "{'ok':'true'}" # required for the JQuery magic delete to succede. - else: - abort(404) - - @rest.restrict('GET') - def itemview_vnum(self, silo, id, path, vnum): - # Check to see if embargo is on: - c.silo_name = silo - c.id = id - c.version = vnum - c.path = path - c_silo = ag.granary.get_rdf_silo(silo) - - if not c_silo.exists(id): - # dataset doesn't exist - abort(404) - - item = c_silo.get_item(id) - vnum = str(vnum) - if not vnum in item.manifest['versions']: - abort(404) - item.set_version_cursor(vnum) - - embargoed = False - if item.metadata.get('embargoed') not in ["false", 0, False]: - embargoed = True - - if embargoed: - #identity management if item - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - - if item.isfile(path): - fileserve_app = FileApp(item.to_dirpath(path)) - return fileserve_app(request.environ, self.start_response) - elif item.isdir(path): - c.parts = item.list_parts(path, detailed=True) - c.readme_text = None - if "README" in c.parts.keys(): - c.readme_text = get_readme_text(item, "%s/README" % path) - - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render("/itemview_version.html") - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - returndata = {} - returndata['parts'] = {} - for part in c.parts: - returndata['parts'][part] = serialisable_stat(c.parts[part]) - returndata['readme_text'] = c.readme_text - return simplejson.dumps(returndata) - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops - nothing satisfies - return text/html - return render("/itemview_version.html") - else: - abort(404) diff --git a/rdfdatabank/controllers/doi.py b/rdfdatabank/controllers/doi.py index 52fcede..115a724 100644 --- a/rdfdatabank/controllers/doi.py +++ b/rdfdatabank/controllers/doi.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from pylons import request, response, session, tmpl_context as c, url, app_globals as ag from pylons.controllers.util import abort from pylons.decorators import rest diff --git a/rdfdatabank/controllers/error.py b/rdfdatabank/controllers/error.py index d43e909..82188c3 100644 --- a/rdfdatabank/controllers/error.py +++ b/rdfdatabank/controllers/error.py @@ -1,13 +1,37 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import cgi from paste.urlparser import PkgResourcesParser -from pylons import request +from pylons import request, response, tmpl_context as c from pylons.controllers.util import forward from pylons.middleware import error_document_template from webhelpers.html.builder import literal -from rdfdatabank.lib.base import BaseController +from rdfdatabank.lib.base import BaseController, render +from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse class ErrorController(BaseController): @@ -25,11 +49,44 @@ def document(self): """Render the error document""" resp = request.environ.get('pylons.original_response') content = literal(resp.body) or cgi.escape(request.GET.get('message', '')) - page = error_document_template % \ - dict(prefix=request.environ.get('SCRIPT_NAME', ''), - code=cgi.escape(request.GET.get('code', str(resp.status_int))), - message=content) - return page + code = cgi.escape(request.GET.get('code', str(resp.status_int))) + if 'HTTP_ACCEPT' in request.environ: + try: + accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) + except: + accept_list= [MT("text", "plain")] + if not accept_list: + accept_list= [MT("text", "plain")] + mimetype = accept_list.pop(0) + while(mimetype): + if str(mimetype).lower() in ["text/html", "text/xhtml"]: + #page = error_document_template % \ + #dict(prefix=request.environ.get('SCRIPT_NAME', ''), + # code=code, + # message=content) + #return page + c.code = code + #c.message = content + c.message = cgi.escape(request.GET.get('message', '')) + c.status = resp.status + c.status = c.status.replace(c.code, '').strip() + if not c.message: + c.message = content.replace(resp.status, '').strip() + return render('/error.html') + elif str(mimetype).lower() in ["text/plain", "application/json"]: + response.content_type = 'text/plain; charset="UTF-8"' + response.status_int = resp.status_int + response.status = resp.status + return content + try: + mimetype = accept_list.pop(0) + except IndexError: + mimetype = None + #Whoops nothing satisfies - return text/plain + response.content_type = 'text/plain; charset="UTF-8"' + response.status_int = resp.status_int + response.status = resp.status + return content def img(self, id): """Serve Pylons' stock images""" diff --git a/rdfdatabank/controllers/home.py b/rdfdatabank/controllers/home.py index c356bad..be6e28c 100644 --- a/rdfdatabank/controllers/home.py +++ b/rdfdatabank/controllers/home.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging from rdfdatabank.lib.base import BaseController, render diff --git a/rdfdatabank/controllers/items.py b/rdfdatabank/controllers/items.py index fb49d3c..a3bddbf 100644 --- a/rdfdatabank/controllers/items.py +++ b/rdfdatabank/controllers/items.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import os, time from datetime import datetime, timedelta @@ -8,7 +31,7 @@ from pylons.decorators import rest from rdfdatabank.lib.base import BaseController, render -from rdfdatabank.lib.utils import create_new +from rdfdatabank.lib.utils import create_new, allowable_id2 from rdfdatabank.lib.file_unpack import check_file_mimetype, BadZipfile, get_zipfiles_in_dataset, unpack_zip_item, read_zipfile from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse @@ -111,12 +134,18 @@ def datasetview(self, silo, id): if params.has_key("id") and params['id']: target_dataset_name = params['id'] else: - (head, fn) = os.path.split(params['filename']) - (fn, ext) = os.path.splitext(fn) - target_dataset_name = "%s-%s"%(id,fn) + #(head, fn) = os.path.split(params['filename']) + #(fn, ext) = os.path.splitext(fn) + #target_dataset_name = "%s-%s"%(id,fn) + target_dataset_name = id #step 1: Create / initialize target dataset if not rdfsilo.exists(target_dataset_name): + if not allowable_id2(target_dataset_name): + response.content_type = "text/plain" + response.status_int = 400 + response.status = "400 Bad request. Dataset name not valid" + return "Dataset name can contain only the following characters - %s and has to be more than 1 character"%ag.naming_rule target_dataset = create_new(rdfsilo, target_dataset_name, ident['repoze.who.userid']) response.status_int = 201 response.status = "201 Created" @@ -255,12 +284,18 @@ def itemview(self, silo, id, path): if params.has_key("id") and params['id']: target_dataset_name = params['id'] else: - (head, fn) = os.path.split(path) - (fn, ext) = os.path.splitext(fn) - target_dataset_name = "%s-%s"%(id,fn) + #(head, fn) = os.path.split(path) + #(fn, ext) = os.path.splitext(fn) + #target_dataset_name = "%s-%s"%(id,fn) + target_dataset_name = id #step 1: Create / initialize target dataset if not rdfsilo.exists(target_dataset_name): + if not allowable_id2(target_dataset_name): + response.content_type = "text/plain" + response.status_int = 400 + response.status = "400 Bad request. Dataset name not valid" + return "Dataset name can contain only the following characters - %s and has to be more than 1 character"%ag.naming_rule target_dataset = create_new(rdfsilo, target_dataset_name, ident['repoze.who.userid']) response.status_int = 201 response.status = "201 Created" diff --git a/rdfdatabank/controllers/items_log.py b/rdfdatabank/controllers/items_log.py deleted file mode 100644 index 45131c9..0000000 --- a/rdfdatabank/controllers/items_log.py +++ /dev/null @@ -1,330 +0,0 @@ -# -*- coding: utf-8 -*- -import logging -import os, time -from datetime import datetime, timedelta -import simplejson -from pylons import request, response, session, tmpl_context as c, url, app_globals as ag -from pylons.controllers.util import abort, redirect_to -from pylons.decorators import rest - -from rdfdatabank.lib.base import BaseController, render -from rdfdatabank.lib.utils import create_new -from rdfdatabank.lib.file_unpack import check_file_mimetype, BadZipfile, get_zipfiles_in_dataset, unpack_zip_item, read_zipfile -from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse - -log = logging.getLogger(__name__) - -class ItemsController(BaseController): - def siloview(self, silo): - abort(403, "Forbidden") - - @rest.restrict('GET', 'POST') - def datasetview(self, silo, id): - #tmpl_context variables needed: c.silo_name, c.zipfiles, c.ident, c.id, c.path - c.silo_name = silo - c.id = id - - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - - rdfsilo = ag.granary.get_rdf_silo(silo) - if not rdfsilo.exists(id): - abort (403, "Forbidden") - - dataset = rdfsilo.get_item(id) - - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - hr = "-"*80 + '\n' - f.write(hr) - f.write("%s :: items - datasetview : silo=%s, id=%s\n"%(str(datetime.now()), silo, id)) - - http_method = request.environ['REQUEST_METHOD'] - - f.write("Http method = %s\n"%http_method) - f.close() - - if http_method == "GET": - c.zipfiles = get_zipfiles_in_dataset(dataset) - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render("/list_of_zipfiles.html") - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return simplejson.dumps(dict(c.zipfiles)) - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - #Whoops nothing satisfies - return text/html - return render("/list_of_zipfiles.html") - elif http_method == "POST": - logstr = [] - logstr.append("="*80) - logstr.append("Items - datasetview of %s, %s"%(silo, id)) - params = request.POST - if not (params.has_key("filename") and params['filename']): - abort(400, "You must supply a filename to unpack") - - logstr.append("POSTED filename %s"%params['filename']) - item_real_filepath = dataset.to_dirpath() - target_filepath = "%s/%s"%(item_real_filepath, params['filename']) - if not os.path.isfile(target_filepath): - abort(404, "File to unpack not found") - if not check_file_mimetype(target_filepath, 'application/zip'): - abort(415, "File is not of type application/zip") - - if params.has_key("id") and params['id']: - target_dataset_name = params['id'] - else: - (head, fn) = os.path.split(params['filename']) - (fn, ext) = os.path.splitext(fn) - target_dataset_name = "%s-%s"%(id,fn) - - #step 1: Create / initialize target dataset - tic = time.mktime(time.gmtime()) - if not rdfsilo.exists(target_dataset_name): - target_dataset = create_new(rdfsilo, target_dataset_name, ident['repoze.who.userid']) - else: - target_dataset = rdfsilo.get_item(target_dataset_name) - toc = time.mktime(time.gmtime()) - logstr.append("1. Time to create / initialize target dataset: %d"%(toc-tic)) - - #step 2: Unpack zip item - logstr.append("Start unpacking zip item : %s"%(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()))) - logstr.append("") - logstr = '\n'.join(logstr) - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write(logstr) - f.close() - try: - unpack_zip_item(target_dataset, dataset, params['filename'], rdfsilo, ident['repoze.who.userid']) - except BadZipfile: - abort(400, "Couldn't unpack zipfile") - logstr = [] - logstr.append("End unpacking zip item : %s"%(time.strftime("%d %b %Y %H:%M:%S", time.gmtime()))) - logstr.append("="*80) - logstr.append("") - logstr.append("") - logstr.append("") - logstr = '\n'.join(logstr) - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write(logstr) - f.close() - - target_dataset.sync() - target_dataset.sync() - target_dataset.sync() - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - return "201 Created" - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - # Whoops - nothing satisfies - return text/plain - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - return "201 Created" - - @rest.restrict('GET', 'POST') - def itemview(self, silo, id, path): - #tmpl_context variables needed: c.silo_name, c.zipfile_contents c.ident, c.id, c.path - c.silo_name = silo - c.id = id - c.path = path - - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - - if not path: - abort(400, "You must supply a filename to unpack") - - ident = request.environ.get('repoze.who.identity') - c.ident = ident - granary_list = ag.granary.silos - silos = ag.authz(granary_list, ident) - if silo not in silos: - abort(403, "Forbidden") - - rdfsilo = ag.granary.get_rdf_silo(silo) - if not rdfsilo.exists(id): - abort (403, "Forbidden") - - dataset = rdfsilo.get_item(id) - item_real_filepath = dataset.to_dirpath() - target_filepath = "%s/%s"%(item_real_filepath, path) - #c.parts = dataset.list_parts(detailed=False) - if not dataset.isfile(path): - abort(404, "File not found") - if not os.path.isfile(target_filepath): - abort(404, "File not found") - if not check_file_mimetype(target_filepath, 'application/zip'): - abort(415, "File is not of type application/zip") - - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - hr = "-"*80 + '\n' - f.write(hr) - f.write("%s :: datasets - datasetview : silo=%s, id=%s\n"%(str(datetime.now()), silo, id)) - - http_method = request.environ['REQUEST_METHOD'] - - f.write("Http method = %s\n"%http_method) - f.close() - - if http_method == "GET": - try: - c.zipfile_contents = read_zipfile(target_filepath) - except BadZipfile: - abort(400, "Could not read zipfile") - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - return render("/zipfileview.html") - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = 'application/json; charset="UTF-8"' - response.status_int = 200 - response.status = "200 OK" - return simplejson.dumps(c.zipfile_contents) - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - # Whoops - nothing satisfies - return text/html - return render("/zipfileview.html") - elif http_method == "POST": - params = request.POST - #if not (params.has_key("filename") and params['filename']): - # abort(400, "You must supply a filename to unpack") - - if params.has_key("id") and params['id']: - target_dataset_name = params['id'] - else: - (head, fn) = os.path.split(path) - (fn, ext) = os.path.splitext(fn) - target_dataset_name = "%s-%s"%(id,fn) - #target_dataset_name, current_dataset, post_filepath, silo, ident - try: - unpack_zip_item(target_dataset_name, dataset, path, rdfsilo, ident['repoze.who.userid']) - except BadZipfile: - abort(400, "Couldn't unpack zipfile") - target_dataset.sync() - target_dataset.sync() - target_dataset.sync() - - # conneg return - accept_list = None - if 'HTTP_ACCEPT' in request.environ: - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Received accept list: %s\n"%str(request.environ['HTTP_ACCEPT'])) - f.close() - try: - accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) - except: - accept_list= [MT("text", "html")] - if not accept_list: - accept_list= [MT("text", "html")] - f = open('/opt/rdfdatabank/src/logs/runtimes_item.log', 'a') - f.write("Parsed accept list: %s\n"%str(accept_list)) - f.close() - mimetype = accept_list.pop(0) - while(mimetype): - if str(mimetype).lower() in ["text/html", "text/xhtml"]: - redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - elif str(mimetype).lower() in ["text/plain", "application/json"]: - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers['Content-Location'] = url(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - return "201 Created" - try: - mimetype = accept_list.pop(0) - except IndexError: - mimetype = None - # Whoops - nothing satisfies - return text/plain - response.content_type = "text/plain" - response.status_int = 201 - response.status = "201 Created" - response.headers["Content-Location"] = url(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name) - return "201 Created" - - @rest.restrict('GET') - def subitemview(self, silo, id, path, subpath): - #tmpl_context variables needed: c.silo_name, c.zipfile_contents c.ident, c.id, c.path - c.silo_name = silo - c.id = id - c.path = path - c.subpath = subpath - - if not request.environ.get('repoze.who.identity'): - abort(401, "Not Authorised") - - if not (path or subpath): - abort(400, "You must supply a filename to unpack") - - return render("/zipfilesubitemview.html") - diff --git a/rdfdatabank/controllers/keywords.py b/rdfdatabank/controllers/keywords.py new file mode 100644 index 0000000..b3f25af --- /dev/null +++ b/rdfdatabank/controllers/keywords.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import logging + +from rdfdatabank.lib.base import BaseController, render + +class KeywordsController(BaseController): + def index(self): + return render('/keywords.html') diff --git a/rdfdatabank/controllers/redirect.py b/rdfdatabank/controllers/redirect.py deleted file mode 100644 index 444e142..0000000 --- a/rdfdatabank/controllers/redirect.py +++ /dev/null @@ -1,21 +0,0 @@ -# -*- coding: utf-8 -*- -import logging -from pylons import url - -from pylons.controllers.util import redirect -from rdfdatabank.lib.base import BaseController - -class RedirectController(BaseController): - def index(self, id): - if id.lower().endswith(('.html', '.rdf', '.json')): - id = id.rsplit('.', 1)[0] - lid = id.lower() - if lid == 'dataset%3a1' or lid == 'dataset:1': - redirect(url(controller="datasets", action="datasetview", silo="general", id='Tick1AudioCorpus')) - elif lid == 'dataset%3A2.html' or lid == 'dataset:2': - redirect(url(controller="datasets", action="datasetview", silo="general", id='RobertDarnton')) - if lid == 'dataset%3A3' or lid == 'dataset:3': - redirect(url(controller="datasets", action="datasetview", silo="general", id='MostynBrown')) - else: - redirect(url(controller="datasets", action="datasetview", silo="general", id=id)) - diff --git a/rdfdatabank/controllers/search.py b/rdfdatabank/controllers/search.py index 83eae72..455b9ee 100644 --- a/rdfdatabank/controllers/search.py +++ b/rdfdatabank/controllers/search.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging from urllib import urlencode, unquote, quote import json @@ -100,8 +123,10 @@ def raw(self): params['start'] = '0' if not 'rows' in params or not params['rows']: params['rows'] = '100' - - result = ag.solr.raw_query(**params) + try: + result = ag.solr.raw_query(**params) + except: + result = {} mimetype = accept_list.pop(0) while(mimetype): @@ -142,7 +167,7 @@ def detailed(self, query=None, additional_fields=[]): if request.params.get("type", None): c.typ = request.params.get("type") - if not c.q or c.q == '*': + if not c.q or c.q == '*' or c.q == "": c.q = "*:*" # Search controls @@ -152,8 +177,34 @@ def detailed(self, query=None, additional_fields=[]): sort = request.params.get('sort', None) format = request.params.get('format', None) if not format: - format = 'html' - + accept_list = None + if 'HTTP_ACCEPT' in request.environ: + try: + accept_list = conneg_parse(request.environ['HTTP_ACCEPT']) + except: + accept_list= [MT("text", "html")] + if not accept_list: + accept_list= [MT("text", "html")] + mimetype = accept_list.pop(0) + while(mimetype): + if str(mimetype).lower() in ["text/html", "text/xhtml"]: + format = 'html' + break + elif str(mimetype).lower() in ["text/plain", "application/json"]: + format = 'json' + break + elif str(mimetype).lower() in ["text/xml"]: + format = 'xml' + break + elif str(mimetype).lower() in ["text/csv"]: + format = 'csv' + break + try: + mimetype = accept_list.pop(0) + except IndexError: + mimetype = None + # Whoops - nothing satisfies - return text/plain + format = 'json' c.sort = 'score desc' # Lock down the sort parameter. @@ -277,17 +328,40 @@ def detailed(self, query=None, additional_fields=[]): solr_params['facet.field'] = [] for facet in c.fields_to_facet: solr_params['facet.field'].append(facet) - - solr_response = ag.solr.raw_query(**solr_params) - + + solr_response = None + try: + solr_response = ag.solr.raw_query(**solr_params) + except: + pass + c.add_facet = u"%ssearch/detailed?q=%s&" % (ag.root, c.q.encode('utf-8')) c.add_facet = c.add_facet + urlencode(c.search) + filter_url if not solr_response: - # FAIL - do something here: - c.message = 'Sorry, either that search "%s" resulted in no matches, or the search service is not functional.' % c.q - h.redirect_to(controller='/search', action='index') + # conneg return + response.status_int = 200 + response.status = "200 OK" + if format == "html": + c.numFound = 0 + c.message = 'Sorry, either that search "%s" resulted in no matches, or the search service is not functional.' % c.q + return render('/search.html') + elif format == 'xml': + response.headers['Content-Type'] = 'application/xml' + response.charset = 'utf8' + c.atom = {} + return render('/atom_results.html') + elif format == 'json': + response.headers['Content-Type'] = 'application/json' + response.charset = 'utf8' + return {} + else: + response.headers['Content-Type'] = 'application/text' + response.charset = 'utf8' + return solr_response + response.status_int = 200 + response.status = "200 OK" if format == 'xml': response.headers['Content-Type'] = 'application/xml' response.charset = 'utf8' @@ -403,8 +477,10 @@ def advanced(self): solr_params['facet.field'] = [] for facet in c.fields_to_facet: solr_params['facet.field'].append(facet) - - solr_response = ag.solr.raw_query(**solr_params) + try: + solr_response = ag.solr.raw_query(**solr_params) + except: + solr_response = None c.add_facet = u"%ssearch/detailed?q=%s&" % (ag.root, c.q.encode('utf-8')) c.add_facet = c.add_facet + urlencode(c.search) + filter_url diff --git a/rdfdatabank/controllers/searching.py b/rdfdatabank/controllers/searching.py new file mode 100644 index 0000000..d215996 --- /dev/null +++ b/rdfdatabank/controllers/searching.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import logging + +from rdfdatabank.lib.base import BaseController, render + +class SearchingController(BaseController): + def index(self): + return render('/searching.html') diff --git a/rdfdatabank/controllers/silos.py b/rdfdatabank/controllers/silos.py index 42100e7..5c2dbed 100644 --- a/rdfdatabank/controllers/silos.py +++ b/rdfdatabank/controllers/silos.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging from datetime import datetime, timedelta import re diff --git a/rdfdatabank/controllers/states.py b/rdfdatabank/controllers/states.py index 16e619a..9ec810c 100644 --- a/rdfdatabank/controllers/states.py +++ b/rdfdatabank/controllers/states.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import simplejson diff --git a/rdfdatabank/controllers/users.py b/rdfdatabank/controllers/users.py index dd399b0..a8ea59e 100644 --- a/rdfdatabank/controllers/users.py +++ b/rdfdatabank/controllers/users.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import simplejson import codecs @@ -85,6 +108,7 @@ def index(self, silo_name): if params['username'] in ag.users: abort(403, "User exists") code = 201 + ag.users[params['username']] = {'role':'', 'first_name':'', 'last_name':'', 'name':'', 'owner':''} owner_of_silos = [] if 'owner' in params and params['owner'] and (('first_name' in params and 'last_name' in params) or 'name' in params) and \ 'username' in params and params['username'] and 'password' in params and params['password']: @@ -230,6 +254,7 @@ def userview(self, silo_name, username): code = 204 else: code = 201 + ag.users[params['username']] = {'role':'', 'first_name':'', 'last_name':'', 'name':'', 'owner':''} owner_of_silos = [] if code == 201: if 'owner' in params and params['owner'] and 'password' in params and params['password'] and \ diff --git a/rdfdatabank/lib/HTTP_request.py b/rdfdatabank/lib/HTTP_request.py index 9ba235d..ad2a63c 100644 --- a/rdfdatabank/lib/HTTP_request.py +++ b/rdfdatabank/lib/HTTP_request.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import logging import mimetypes import httplib diff --git a/rdfdatabank/lib/app_globals.py b/rdfdatabank/lib/app_globals.py index 61575a8..ce3425a 100644 --- a/rdfdatabank/lib/app_globals.py +++ b/rdfdatabank/lib/app_globals.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """The application's Globals object""" from pylons import config @@ -11,6 +34,7 @@ from rdfdatabank.lib.broadcast import BroadcastToRedis from rdfdatabank.config.users import _USERS +from rdfdatabank.config.namespaces import NAMESPACES, PREFIXES class Globals(object): @@ -28,6 +52,8 @@ def __init__(self): self.authz = authz self.users = _USERS + self.NAMESPACES = NAMESPACES + self.PREFIXES = PREFIXES if config.has_key("granary.uri_root"): self.root = config['granary.uri_root'] @@ -37,17 +63,28 @@ def __init__(self): if config.has_key("redis.host"): self.redishost = config['redis.host'] - self.r = Redis(self.redishost) + try: + self.r = Redis(self.redishost) + except: + self.r = None + if self.r and config.has_key("broadcast.to") and config['broadcast.to'] == "redis" and config.has_key("broadcast.queue"): + self.b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) + else: + self.r = None + self.redishost = None + self.b = None if config.has_key("solr.host"): from solr import SolrConnection self.solrhost = config['solr.host'] - self.solr = SolrConnection(self.solrhost) + try: + self.solr = SolrConnection(self.solrhost) + except: + self.solr = None + else: + self.solrhost = None + self.solr = None - if config.has_key("broadcast.to"): - if config['broadcast.to'] == "redis": - self.b = BroadcastToRedis(config['redis.host'], config['broadcast.queue']) - if config.has_key("naming_rule"): self.naming_rule = config['naming_rule'] @@ -90,3 +127,6 @@ def __init__(self): if config.has_key("license"): self.license = config['license'] + if config.has_key("api.version"): + self.api_version = config['api.version'] + diff --git a/rdfdatabank/lib/auth.py b/rdfdatabank/lib/auth.py index 05051f2..1a1acde 100644 --- a/rdfdatabank/lib/auth.py +++ b/rdfdatabank/lib/auth.py @@ -1,3 +1,27 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from webob import Request import zope.interface from repoze.who.classifiers import default_request_classifier diff --git a/rdfdatabank/lib/base.py b/rdfdatabank/lib/base.py index 792d640..351464a 100644 --- a/rdfdatabank/lib/base.py +++ b/rdfdatabank/lib/base.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """The base Controller API Provides the BaseController class for subclassing. diff --git a/rdfdatabank/lib/broadcast.py b/rdfdatabank/lib/broadcast.py index 73ab39b..3025e21 100644 --- a/rdfdatabank/lib/broadcast.py +++ b/rdfdatabank/lib/broadcast.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from redis import Redis from redis.exceptions import ConnectionError diff --git a/rdfdatabank/lib/conneg.py b/rdfdatabank/lib/conneg.py index d822606..8cc0bad 100644 --- a/rdfdatabank/lib/conneg.py +++ b/rdfdatabank/lib/conneg.py @@ -1,5 +1,28 @@ -# -*- coding: utf-8 -*- #!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from pylons import app_globals as ag from datetime import datetime def skipws(next): diff --git a/rdfdatabank/lib/doi_helper.py b/rdfdatabank/lib/doi_helper.py index 86e4eef..5cd5ef9 100644 --- a/rdfdatabank/lib/doi_helper.py +++ b/rdfdatabank/lib/doi_helper.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from rdfdatabank.lib.doi_schema import DataciteDoiSchema from pylons import app_globals as ag import os, codecs, uuid diff --git a/rdfdatabank/lib/doi_schema.py b/rdfdatabank/lib/doi_schema.py index 623f5cb..1619ad1 100755 --- a/rdfdatabank/lib/doi_schema.py +++ b/rdfdatabank/lib/doi_schema.py @@ -1,4 +1,28 @@ +#!/usr/bin/python # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + class DataciteDoiSchema(): def __init__(self): """ diff --git a/rdfdatabank/lib/file_unpack.py b/rdfdatabank/lib/file_unpack.py index 268dce3..0ac4dca 100644 --- a/rdfdatabank/lib/file_unpack.py +++ b/rdfdatabank/lib/file_unpack.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + import subprocess from threading import Thread from datetime import datetime, timedelta @@ -47,7 +70,8 @@ def get_zipfiles_in_dataset(dataset): real_filepath = os.readlink(real_filepath) if check_file_mimetype(real_filepath, 'application/zip'): (fn, ext) = os.path.splitext(filepath) - zipfiles[filepath]="%s-%s"%(dataset.item_id, fn) + #zipfiles[filepath]="%s-%s"%(dataset.item_id, fn) + zipfiles[filepath]=dataset.item_id return zipfiles def store_zipfile(silo, target_item_uri, POSTED_file, ident): @@ -139,7 +163,6 @@ def unpack_zip_item(target_dataset, current_dataset, zip_item, silo, ident): filepath = current_dataset.to_dirpath(zip_item) if os.path.islink(filepath): filepath = os.readlink(filepath) - emb = target_dataset.metadata.get('embargoed') emb_until = target_dataset.metadata.get('embargoed_until') @@ -150,7 +173,7 @@ def unpack_zip_item(target_dataset, current_dataset, zip_item, silo, ident): file_uri = current_dataset.uri if not file_uri.endswith('/'): file_uri += '/' - file_uri = "%s%s"%(file_uri,zip_item) + file_uri = "%s%s?version=%s"%(file_uri,zip_item,current_dataset.currentversion) items_list = [] os.path.walk(unpacked_dir,get_items_in_dir,items_list) diff --git a/rdfdatabank/lib/file_unpack_log.py b/rdfdatabank/lib/file_unpack_log.py deleted file mode 100644 index dd11bbd..0000000 --- a/rdfdatabank/lib/file_unpack_log.py +++ /dev/null @@ -1,253 +0,0 @@ -# -*- coding: utf-8 -*- -import subprocess - -from datetime import datetime, timedelta -import time -import os - -from redis import Redis - -from uuid import uuid4 - -from rdfdatabank.lib.utils import create_new, munge_manifest, test_rdf - -#import checkm -from zipfile import ZipFile, BadZipfile as BZ - -zipfile_root = "zipfile:" - -class BadZipfile(Exception): - """Cannot open zipfile using commandline tool 'unzip' to target directory""" - -def check_file_mimetype(real_filepath, mimetype): - if os.path.islink(real_filepath): - real_filepath = os.readlink(real_filepath) - p = subprocess.Popen("file -ib %s" %(real_filepath), shell=True, stdout=subprocess.PIPE) - output_file = p.stdout - output_str = output_file.read() - if mimetype in output_str: - return True - else: - return False - -def get_zipfiles_in_dataset(dataset): - derivative = dataset.list_rdf_objects("*", "ore:aggregates") - #print "derivative ", derivative - #print "derivative values", derivative.values()[0] - zipfiles = {} - if derivative and derivative.values() and derivative.values()[0]: - for file_uri in derivative.values()[0]: - filepath = file_uri[len(dataset.uri)+1:] - real_filepath = dataset.to_dirpath(filepath) - if os.path.islink(real_filepath): - real_filepath = os.readlink(real_filepath) - #print "file_uri : ", file_uri - #print "filepath : ", filepath - #print "real_filepath : ", real_filepath - if check_file_mimetype(real_filepath, 'application/zip'): - (fn, ext) = os.path.splitext(filepath) - zipfiles[filepath]="%s-%s"%(dataset.item_id, fn) - return zipfiles - -def store_zipfile(silo, target_item_uri, POSTED_file, ident): - zipfile_id = get_next_zipfile_id(silo.state['storage_dir']) - while(silo.exists("%s%s" % (zipfile_root, zipfile_id))): - zipfile_id = get_next_zipfile_id(silo.state['storage_dir']) - - #zip_item = silo.get_item("%s%s" % (zipfile_root, zipfile_id)) - zip_item = create_new(silo, "%s%s" % (zipfile_root, zipfile_id), ident) - zip_item.add_triple("%s/%s" % (zip_item.uri, POSTED_file.filename.lstrip(os.sep)), "dcterms:hasVersion", target_item_uri) - zip_item.put_stream(POSTED_file.filename, POSTED_file.file) - try: - POSTED_file.file.close() - except: - pass - zip_item.sync() - return zip_item - -def read_zipfile(filepath): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - - # list filenames - #list_of_files = tmpfile.namelist() - - # file information - zipfile_contents = {} - for info in tmpfile.infolist(): - zipfile_contents[info.filename] = (info.file_size, info.date_time) - tmpfile.close() - return zipfile_contents - -def read_file_in_zipfile(filepath, filename): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - - try: - fileinfo = tmpfile.getinfo(filename) - except KeyError: - return False - if fileinfo.file_size == 0: - return 0 - - # read file - file_contents = None - file_contents = tmpfile.read(filename) - tmpfile.close() - return file_contents - -def get_file_in_zipfile(filepath, filename, targetdir): - try: - tmpfile = ZipFile(filepath, "r") - except BZ: - raise BadZipfile - - try: - fileinfo = tmpfile.getinfo(filename) - except KeyError: - return False - if fileinfo.file_size == 0: - return 0 - - # extract file - targetfile = tmpfile.extract(filename, targetdir) - tmpfile.close() - return targetfile - -def unzip_file(filepath, target_directory=None): - #f = open("/tmp/python_out.log", "a") - #f.write("\n--------------- In unzip file -------------------\n") - #f.write("filepath : %s\n"%str(filepath)) - #f.write('-'*40+'\n') - #f.close() - # TODO add the checkm stuff back in - if not target_directory: - target_directory = "/tmp/%s" % (uuid4().hex) - p = subprocess.Popen("unzip -qq -d %s %s" % (target_directory, filepath), shell=True, stdout=subprocess.PIPE) - p.wait() - if p.returncode != 0: - raise BadZipfile - else: - return target_directory - -def get_items_in_dir(items_list, dirname, fnames): - for fname in fnames: - items_list.append(os.path.join(dirname,fname)) - return - -def unpack_zip_item(target_dataset, current_dataset, zip_item, silo, ident): - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - hr = "-"*80 + '\n' - f.write(hr) - f.write("file_unpack - unpack_zip_item\n") - f.write("Unpacking %s in %s TO %s\n"%(zip_item, current_dataset.uri, target_dataset.uri)) - filepath = current_dataset.to_dirpath(zip_item) - if os.path.islink(filepath): - filepath = os.readlink(filepath) - - # -- Step 1 ----------------------------- - tic1 = time.mktime(time.gmtime()) - unpacked_dir = unzip_file(filepath) - toc = time.mktime(time.gmtime()) - f.write("1. Time to unpack: %d\n"%(toc-tic1)) - f.close() - - # -- Step 2 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - tic = time.mktime(time.gmtime()) - file_uri = current_dataset.uri - if not file_uri.endswith('/'): - file_uri += '/' - file_uri = "%s%s"%(file_uri,zip_item) - - items_list = [] - os.path.walk(unpacked_dir,get_items_in_dir,items_list) - toc = time.mktime(time.gmtime()) - f.write("2. Time to walk: %d\n"%(toc-tic)) - f.close() - - # -- Step 3 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - tic = time.mktime(time.gmtime()) - manifest_str = None - #Read manifest - for i in items_list: - if 'manifest.rdf' in i and os.path.isfile(i): - F = open(i, 'r') - manifest_str = F.read() - F.close() - items_list.remove(i) - os.remove(i) - break - toc = time.mktime(time.gmtime()) - f.write("3. Time to read manifest: %d\n"%(toc-tic)) - f.close() - - # -- Step 4 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - tic = time.mktime(time.gmtime()) - #Copy unpacked dir as new version - target_dataset.move_directory_as_new_version(unpacked_dir) - toc = time.mktime(time.gmtime()) - f.write("4. Time to move unpacked file: %d\n"%(toc-tic)) - f.close() - - # -- Step 5 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - tic = time.mktime(time.gmtime()) - #Add type and isVersionOf metadata - target_dataset.add_namespace('oxds', "http://vocab.ox.ac.uk/dataset/schema#") - target_dataset.add_triple(target_dataset.uri, u"rdf:type", "oxds:Grouping") - target_dataset.add_triple(target_dataset.uri, "dcterms:isVersionOf", file_uri) - #TODO: Adding the following metadata again as moving directory deletes all this information. Need to find a better way - embargoed_until_date = (datetime.now() + timedelta(days=365*70)).isoformat() - target_dataset.add_triple(target_dataset.uri, u"oxds:isEmbargoed", 'True') - target_dataset.add_triple(target_dataset.uri, u"oxds:embargoedUntil", embargoed_until_date) - target_dataset.add_triple(target_dataset.uri, u"dcterms:identifier", target_dataset.item_id) - target_dataset.add_triple(target_dataset.uri, u"dcterms:creator", ident) - target_dataset.add_triple(target_dataset.uri, u"dcterms:created", datetime.now()) - target_dataset.add_triple(target_dataset.uri, u"oxds:currentVersion", target_dataset.currentversion) - #Adding ore aggregates - unp_dir = unpacked_dir - if not unp_dir.endswith('/'): - unp_dir += '/' - target_uri_base = target_dataset.uri - if not target_uri_base.endswith('/'): - target_uri_base += '/' - for i in items_list: - i = i.replace(unp_dir, '') - target_dataset.add_triple(target_dataset.uri, "ore:aggregates", "%s%s"%(target_uri_base,i)) - target_dataset.add_triple(target_dataset.uri, u"dcterms:modified", datetime.now()) - target_dataset.sync() - toc = time.mktime(time.gmtime()) - f.write("5. Time to add metadata: %d\n"%(toc-tic)) - f.close() - - # -- Step 6 ----------------------------- - f = open('/opt/rdfdatabank/src/logs/runtimes_items.log', 'a') - tic = time.mktime(time.gmtime()) - #Munge rdf - #TODO: If manifest is not well formed rdf - inform user. Currently just ignored. - if manifest_str and test_rdf(manifest_str): - munge_manifest(manifest_str, target_dataset, manifest_type='http://vocab.ox.ac.uk/dataset/schema#Grouping') - toc = time.mktime(time.gmtime()) - f.write("6. Time to munge rdf: %d\n"%(toc-tic)) - - # -- Step 7 ----------------------------- - tic = time.mktime(time.gmtime()) - target_dataset.sync() - target_dataset.sync() - target_dataset.sync() - current_dataset.add_triple("%s/%s" % (current_dataset.uri, zip_item.lstrip(os.sep)), "dcterms:hasVersion", target_dataset.uri) - current_dataset.sync() - toc = time.mktime(time.gmtime()) - f.write("7. Time to Sync and unpadte parent manifest: %d\n"%(toc-tic)) - - f.write(hr) - f.write("Total Time: %d\n\n"%(toc-tic1)) - f.close() - return True diff --git a/rdfdatabank/lib/helpers.py b/rdfdatabank/lib/helpers.py index e18f53b..aff6947 100644 --- a/rdfdatabank/lib/helpers.py +++ b/rdfdatabank/lib/helpers.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + """Helper functions Consists of functions to typically be used within templates, but also diff --git a/rdfdatabank/lib/htpasswd.py b/rdfdatabank/lib/htpasswd.py index 1a46527..e4da240 100644 --- a/rdfdatabank/lib/htpasswd.py +++ b/rdfdatabank/lib/htpasswd.py @@ -1,8 +1,38 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -"""Replacement for htpasswd""" -#Downloaded from: http://trac.edgewall.org/browser/trunk/contrib/htpasswd.py -# Original author: Eli Carter +"""Replacement for htpasswd +Downloaded from: http://trac.edgewall.org/browser/trunk/contrib/htpasswd.py +Original author: Eli Carter + +Copyright (C) 2003-2012 Edgewall Software +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The name of the author may not be used to endorse or promote + products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER +IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +""" import os import sys diff --git a/rdfdatabank/lib/ident_md.py b/rdfdatabank/lib/ident_md.py index 22a4324..d7fc87a 100644 --- a/rdfdatabank/lib/ident_md.py +++ b/rdfdatabank/lib/ident_md.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from rdfdatabank.config.users import _USERS as _DATA class IdentMDProvider(object): diff --git a/rdfdatabank/lib/search_term.py b/rdfdatabank/lib/search_term.py index 85fbae0..9ae3737 100644 --- a/rdfdatabank/lib/search_term.py +++ b/rdfdatabank/lib/search_term.py @@ -1,3 +1,28 @@ +# -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + + class term_list(): def get_all_search_fields(self): return [ diff --git a/rdfdatabank/lib/short_pid.py b/rdfdatabank/lib/short_pid.py index d0872da..d46e946 100644 --- a/rdfdatabank/lib/short_pid.py +++ b/rdfdatabank/lib/short_pid.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- +""" #Downloaded from http://code.activestate.com/recipes/576918/ #Created by Michael Fogleman #Short URL Generator +""" #DEFAULT_ALPHABET = 'JedR8LNFY2j6MrhkBSADUyfP5amuH9xQCX4VqbgpsGtnW7vc3TwKE' #DEFAULT_BLOCK_SIZE = 22 diff --git a/rdfdatabank/lib/utils.py b/rdfdatabank/lib/utils.py index f483979..565a5f1 100644 --- a/rdfdatabank/lib/utils.py +++ b/rdfdatabank/lib/utils.py @@ -1,4 +1,27 @@ # -*- coding: utf-8 -*- +""" +Copyright (c) 2012 University of Oxford + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, --INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + from datetime import datetime, timedelta from dateutil.relativedelta import * from dateutil.parser import parse @@ -13,9 +36,9 @@ from rdflib import StringInputSource from rdflib import Namespace, RDF, RDFS, URIRef, Literal, BNode - from uuid import uuid4 import re +from collections import defaultdict ID_PATTERN = re.compile(r"^[0-9A-z\-\:]+$") @@ -82,8 +105,11 @@ def is_embargoed(silo, id, refresh=False): # TODO evaluate ag.r.expire settings for these keys - popularity resets ttl or increases it? e = None e_d = None - e = ag.r.get("%s:%s:embargoed" % (silo.state['storage_dir'], id)) - e_d = ag.r.get("%s:%s:embargoed_until" % (silo.state['storage_dir'], id)) + try: + e = ag.r.get("%s:%s:embargoed" % (silo.state['storage_dir'], id)) + e_d = ag.r.get("%s:%s:embargoed_until" % (silo.state['storage_dir'], id)) + except: + pass if refresh or (not e or not e_d): if silo.exists(id): @@ -94,11 +120,20 @@ def is_embargoed(silo, id, refresh=False): e = True else: e = False - ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), e) - ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d) + try: + ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), e) + ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d) + except: + pass return (e, e_d) def get_embargo_values(embargoed=None, embargoed_until=None, embargo_days_from_now=None): + if isinstance(embargoed, basestring): + embargoed = embargoed.strip() + if isinstance(embargoed_until, basestring): + embargoed_until = embargoed_until.strip() + if isinstance(embargo_days_from_now, basestring): + embargo_days_from_now = embargo_days_from_now.strip() e_status=None e_date=None if embargoed == None: @@ -137,20 +172,32 @@ def create_new(silo, id, creator, title=None, embargoed=None, embargoed_until=No item.metadata['embargoed_until'] = '' item.del_triple(item.uri, u"oxds:isEmbargoed") item.del_triple(item.uri, u"oxds:embargoedUntil") - ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), ' ') + try: + ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), ' ') + except: + pass e, e_d = get_embargo_values(embargoed=embargoed, embargoed_until=embargoed_until, embargo_days_from_now=embargo_days_from_now) if e: item.metadata['embargoed'] = True item.add_triple(item.uri, u"oxds:isEmbargoed", 'True') - ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), True) + try: + ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), True) + except: + pass if e_d: item.metadata['embargoed_until'] = e_d - item.add_triple(item.uri, u"oxds:embargoedUntil", e_d) - ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d) + item.add_triple(item.uri, u"oxds:embargoedUntil", e_d) + try: + ag.r.set("%s:%s:embargoed_until" % (silo.state['storage_dir'], id), e_d) + except: + pass else: item.metadata['embargoed'] = False item.add_triple(item.uri, u"oxds:isEmbargoed", 'False') - ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), False) + try: + ag.r.set("%s:%s:embargoed" % (silo.state['storage_dir'], id), False) + except: + pass item.add_triple(item.uri, u"dcterms:identifier", id) item.add_triple(item.uri, u"dcterms:mediator", creator) @@ -329,4 +376,61 @@ def serialisable_stat(stat): pass return stat_values +def natural_sort(l): + convert = lambda text: int(text) if text.isdigit() else text.lower() + alphanum_key = lambda key: [ convert(c) for c in re.split('([0-9]+)', key) ] + return sorted(l, key = alphanum_key) + +def extract_metadata(item): + g = item.get_graph() + m = defaultdict(list) + #for s,p,o in g.triples((URIRef(item.uri), ag.NAMESPACES[dc]['identifier'], None)): + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dc']['title']): + m['title'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dc']['identifier']): + m['identifier'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dc']['description']): + m['description'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dc']['creator']): + m['creator'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dc']['subject']): + m['subject'].append(o) + + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['abstract']): + m['abstract'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['created']): + m['created'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['description']): + m['description'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['hasVersion']): + m['hasVersion'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['identifier']): + m['identifier'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['isVersionOf']): + m['isVersionOf'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['license']): + m['license'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['mediator']): + m['mediator'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['modified']): + m['modified'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['publisher']): + m['publisher'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['rights']): + m['rights'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['subject']): + m['subject'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['dcterms']['title']): + m['title'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['oxds']['isEmbargoed']): + m['isEmbargoed'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['oxds']['embargoedUntil']): + m['embargoedUntil'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['oxds']['currentVersion']): + m['currentVersion'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['bibo']['doi']): + m['doi'].append(o) + for o in g.objects(URIRef(item.uri), ag.NAMESPACES['ore']['aggregates']): + m['aggregates'].append(o) + return dict(m) diff --git a/rdfdatabank/public/static/js/html5.js b/rdfdatabank/public/static/js/html5.js new file mode 100644 index 0000000..6692336 --- /dev/null +++ b/rdfdatabank/public/static/js/html5.js @@ -0,0 +1,4 @@ +// iepp v2.1pre @jon_neal & @aFarkas github.com/aFarkas/iepp +// html5shiv @rem remysharp.com/html5-enabling-script +// Dual licensed under the MIT or GPL Version 2 licenses +/*@cc_on(function(a,b){function r(a){var b=-1;while(++b +<%def name="head_tags()"> + About DataBank + + +

info on project goes here goes here...

+ +

Background

+ + +

Documentation

+ + + \ No newline at end of file diff --git a/rdfdatabank/templates/admin_siloview.html b/rdfdatabank/templates/admin_siloview.html index f53191c..ce4a9ec 100644 --- a/rdfdatabank/templates/admin_siloview.html +++ b/rdfdatabank/templates/admin_siloview.html @@ -1,20 +1,22 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Admin Interface - ${c.silo_name} - -

Admin Interface - ${c.silo_name}

-% if c.silo_name: -

Admin ${c.silo_name}

-% if c.message: -

${c.message}

-% endif -
-

Delete this silo - NO UNDO!

-
-% endif -<%include file="/silo_metadata.html"/> -
-<%include file="/alter_silo.html"/> -
- +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Admin Interface - ${c.silo_name} + +

Admin Interface - ${c.silo_name}

+% if c.silo_name: +

Admin ${c.silo_name}

+% if c.message: +

${c.message}

+% endif +
+

Delete this silo - NO UNDO!

+
+% endif +

Silo metadata

+<%include file="/silo_metadata.html"/> +
+

Update silo metadata

+<%include file="/alter_silo.html"/> +
+ diff --git a/rdfdatabank/templates/admin_user.html b/rdfdatabank/templates/admin_user.html index 182e637..7e3c178 100644 --- a/rdfdatabank/templates/admin_user.html +++ b/rdfdatabank/templates/admin_user.html @@ -1,37 +1,38 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - User Info - ${c.silo_name} - -

Silo ${c.silo_name}

-

User info for ${c.username}

-% if c.message: -

${c.message}

-% endif -%if c.user: - -% endif -
-<%include file="/update_user.html"/> +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + User Info - ${c.silo_name} + +

Silo ${c.silo_name}

+

User info for ${c.username}

+% if c.message: +

${c.message}

+% endif +%if c.user: + +% endif +
+

Update user information

+<%include file="/update_user.html"/> diff --git a/rdfdatabank/templates/admin_users.html b/rdfdatabank/templates/admin_users.html index 7db49d8..fff3cc7 100644 --- a/rdfdatabank/templates/admin_users.html +++ b/rdfdatabank/templates/admin_users.html @@ -1,20 +1,20 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Users - ${c.silo_name} - -

Users - ${c.silo_name}

-

Users in ${c.silo_name}

-% if c.message: -

${c.message}

-% endif -
-% if c.users: -
    - % for o in c.users: -
  • ${o}
  • - % endfor -
-% endif -
-<%include file="/register_new_user.html"/> +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Users - ${c.silo_name} + +

Users of ${c.silo_name}

+% if c.message: +

${c.message}

+% endif +
+% if c.users: +
    + % for o in c.users: +
  • ${o}
  • + % endfor +
+% endif +
+

Create new user

+<%include file="/register_new_user.html"/> diff --git a/rdfdatabank/templates/alter_silo.html b/rdfdatabank/templates/alter_silo.html index f83250d..8d06f3a 100644 --- a/rdfdatabank/templates/alter_silo.html +++ b/rdfdatabank/templates/alter_silo.html @@ -1,23 +1,23 @@ -# -*- coding: utf-8 -*- -

Alter Silo

-

-

-
    -
  • ${c.silo_name}
  • -% if c.kw: -
  • -
  • -
  • -
  • -
  • -% else: -
  • -
  • -
  • -
  • -
  • -% endif -
  • -
-
-

+# -*- coding: utf-8 -*- +

+

+
    +
  • ${c.silo_name}

  • +% if c.kw: +
  • +
  • +
  • +
  • +
    List all of the users who can access this silo. Separate usernames with a comma
  • +
  • +% else: +
  • +
  • +
  • +
  • +
  • +% endif +
  • +
+
+

diff --git a/rdfdatabank/templates/api.html b/rdfdatabank/templates/api.html index a740db5..b8da929 100644 --- a/rdfdatabank/templates/api.html +++ b/rdfdatabank/templates/api.html @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Databank API - -

Databank API

-

- -<%include file="${c.api_file}"/> +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Databank API + +

Databank API

+

+ +<%include file="${c.api_file}"/> diff --git a/rdfdatabank/templates/atom_results.html b/rdfdatabank/templates/atom_results.html index 2a8a0a5..ebedd08 100644 --- a/rdfdatabank/templates/atom_results.html +++ b/rdfdatabank/templates/atom_results.html @@ -1,2 +1,2 @@ - -${c.atom} + +${c.atom} diff --git a/rdfdatabank/templates/base.html b/rdfdatabank/templates/base.html index 88c7d63..4f344e2 100644 --- a/rdfdatabank/templates/base.html +++ b/rdfdatabank/templates/base.html @@ -1,99 +1,197 @@ -# -*- coding: utf-8 -*- - - - - - ${ h.stylesheet_link( '/static/style.css' ) } - ${ h.javascript_link( '/static/jquery.js' ) } - ${self.head_tags()} - - - -
- -
- <% - user_logged_in = request.environ.get("repoze.who.identity") - if user_logged_in: - c.user_logged_in_name = user_logged_in['repoze.who.userid'] - current_url = request.environ.get('PATH_INFO') - %> - -
- % if c.id: - % if c.path: -

Subdirectory ${c.id}/${c.path}

-

Root directory: ${c.id}

- % else: -

Root directory of ${c.id}

- % endif - % endif - ${self.body()} -
- -
- - +# -*- coding: utf-8 -*- + + + + + + + + + + + + + + + ${ h.javascript_link( '/static/jquery.js' ) } + ${self.head_tags()} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + +
+ + +
+ +
+ + + + + + + ${self.body()} +
+
+
+ + +
+
+ <%include file="/footer.html"/> +
+
+ + diff --git a/rdfdatabank/templates/create_doi.html b/rdfdatabank/templates/create_doi.html index aa972d8..f98db20 100644 --- a/rdfdatabank/templates/create_doi.html +++ b/rdfdatabank/templates/create_doi.html @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- -
- -
+# -*- coding: utf-8 -*- +
+ +
diff --git a/rdfdatabank/templates/create_new_item.html b/rdfdatabank/templates/create_new_item.html index b020d30..c8cd06a 100644 --- a/rdfdatabank/templates/create_new_item.html +++ b/rdfdatabank/templates/create_new_item.html @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- -
- - -
+# -*- coding: utf-8 -*- +
+ + +
diff --git a/rdfdatabank/templates/create_new_silo.html b/rdfdatabank/templates/create_new_silo.html index 0e481db..5bc61a8 100644 --- a/rdfdatabank/templates/create_new_silo.html +++ b/rdfdatabank/templates/create_new_silo.html @@ -1,15 +1,18 @@ -# -*- coding: utf-8 -*- -

Create new Silo

-

-

-
    -
  • -
  • -
  • -
  • -
  • -
  • -
  • -
-
-

+# -*- coding: utf-8 -*- +
+

Create new Silo

+ +
+
    +
  • +
    The machine readable name of the silo, used by the system and in the urls.

    For example the name siloa will appear in the url as http://databank/siloa/...
    +
  • +
  • +
  • +
  • +
    List all of the users who can access this silo. Separate multiple usernames with a comma
    +
  • +
  • +
+
+
diff --git a/rdfdatabank/templates/datasetview.html b/rdfdatabank/templates/datasetview.html index 3e40b18..f75559d 100644 --- a/rdfdatabank/templates/datasetview.html +++ b/rdfdatabank/templates/datasetview.html @@ -1,114 +1,217 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - - Dataset ${c.id} - - -<% -c.current = c.id -%> - -

Dataset ${c.id}

- - -% if c.embargos: -

Embargo Information

-% if c.embargos[c.current][0] not in ['False', 'false', 0, False]: -

Embargoed: True

-

Embargoed until: ${c.embargos[c.current][1]}

-% else: -

Embargoed: False

-% endif -% endif - - -% if c.editor: -
- % if c.view == 'editor': - Editor view - % else: - User view - % endif - % if c.version: -
Change view : - Editor | - User

- % else: -
Change view : - Editor | - User

- % endif -
-% endif - -
-% if c.version: -

Information for version ${c.version} of the dataset

-% endif - - - -% if c.show_files: - % if c.editor: - <%include file="/part_list.html"/> - % else: - <%include file="/part_list_display.html"/> - % endif -% endif - - -% if c.readme_text: -<%include file="/readme_section.html"/> -% endif - - -
-
Item's RDF Manifest (Accept: most RDF mimetypes):
-
${c.manifest_pretty}
-
- -
- - - -% if c.editor: -
-

View state information

- % if c.view == 'editor': -
- -
Change Embargo
-
- <%include file="/embargo_form.html"/> -

Embargo state: True - only those logged in and with edit rights can see item. False - Anyone can GET the item and it's files.

-

Embargo date: Aim is for ISO8601 dates to provide embargo trigger events. Currently unused, unvalidated and unparsed.

-
- -
Add metadata to the RDF Manifest:
-
<%include file="/rdf_manifest_form.html"/>
-
- - <%include file="/file_upload.html"/> - - % if c.zipfiles: -

List of item unpack endpoints:

-
    - % for zf in c.zipfiles: - <% - c.zipfile = zf - c.newid = c.zipfiles[zf] - %> -
  • <%include file="/files_unpack.html"/>
  • - % endfor -
- % endif - % endif -
-% endif - +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + + Dataset ${c.id} + + + + +<% +c.current = c.id +%> +

Dataset ${c.id}

+% if c.version: +

Version ${c.version}

+% elif c.versions: +

Version ${c.versions[-1]}

+% endif + +% if c.editor: + +% endif + + +
+% if c.version and c.versions and c.version == c.versions[-1]: +

You are currently viewing the latest version (${c.version}) of the data package

+% elif c.version: +

You are currently viewing version ${c.version} of the data package

+% elif c.versions: +

You are currently viewing the latest version (${c.versions[-1]}) of the data package

+% else: +

You are currently viewing the latest version of th data package

+% endif + +% if c.versions: +

View other versions: + % for i in range(1, len(c.versions)-1): + % if c.version and c.versions[i] == c.version: + ${c.versions[i]} | + % else: + ${c.versions[i]} | + % endif + % endfor + % if c.version: + % if c.version == c.versions[-1]: + ${c.versions[-1]} + % else: + ${c.versions[-1]} + % endif + % else: + ${c.versions[-1]} + % endif +

+% endif +
+ +
+ + +% if c.embargos: +

Access Information

+
+% if c.embargos and c.current in c.embargos and c.embargos[c.current]: + <% c.emb = c.embargos[c.current] %> + % if c.emb[0] == True or c.emb[0] == 1 or (isinstance(c.emb[0], basestring) and c.emb[0].strip().lower() in ['true', '1']): + % if (isinstance(c.emb[1], basestring) and c.emb[1].strip()): + Data package is embargoed until ${c.emb[1]}. Only the metadata is openly accessible. + % else: + Data package is embargoed indefinitely. Only the metadata is openly accessible. + % endif + % else: + Data package is openly accessible + % endif +% else: + Data package is openly accessible +% endif +% endif +
+ +
+ +% if c.show_files: +

Data package contents

+
+ % if c.editor: + <%include file="/part_list.html"/> + % else: + <%include file="/part_list_display.html"/> + % endif +
+% endif + + +% if c.readme_text: +

README text

+
+<%include file="/readme_section.html"/> +
+% endif + + +

Metadata

+
+ +<%include file="/rdf_manifest.html"/> +
+ +
+ + +% if c.editor: +
+ % if c.view == 'editor': +
+ + + + + + + + + + % if c.zipfiles: + + % endif + % endif +
+% endif + +
diff --git a/rdfdatabank/templates/datasetview_version.html b/rdfdatabank/templates/datasetview_version.html index 6e18250..f7b645c 100644 --- a/rdfdatabank/templates/datasetview_version.html +++ b/rdfdatabank/templates/datasetview_version.html @@ -1,49 +1,49 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - - Dataset ${c.id} - - -<% -c.current = c.id -%> - -

Dataset ${c.id}

- -
- - - -% if c.editor or c.embargos[c.current][0] in ['False', 'false', 0, False]: -<%include file="/part_list_display.html"/> -% endif - - -% if c.readme_text: -<%include file="/readme_section.html"/> -% endif - - -
-% if c.embargos: -
Embargo Information
-% if c.embargos[c.current][0] not in ['False', 'false', 0, False]: -
Embargoed: True
-
Embargoed until: ${c.embargos[c.current][1]}
-% else: -
Embargoed: False
-% endif -% endif -
- - -
Item's RDF Manifest (Accept: most RDF mimetypes):
-
${c.manifest_pretty}
- -
+# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + + Dataset ${c.id} + + +<% +c.current = c.id +%> + +

Dataset ${c.id}

+ +
+ + + +% if c.editor or c.embargos[c.current][0] in ['False', 'false', 0, False]: +<%include file="/part_list_display.html"/> +% endif + + +% if c.readme_text: +<%include file="/readme_section.html"/> +% endif + + +
+% if c.embargos: +
Embargo Information
+% if c.embargos[c.current][0] not in ['False', 'false', 0, False]: +
Embargoed: True
+
Embargoed until: ${c.embargos[c.current][1]}
+% else: +
Embargoed: False
+% endif +% endif +
+ + +
Item's RDF Manifest (Accept: most RDF mimetypes):
+
${c.manifest_pretty}
+ +
diff --git a/rdfdatabank/templates/delete_item.html b/rdfdatabank/templates/delete_item.html index b020d30..c8cd06a 100644 --- a/rdfdatabank/templates/delete_item.html +++ b/rdfdatabank/templates/delete_item.html @@ -1,5 +1,5 @@ -# -*- coding: utf-8 -*- -
- - -
+# -*- coding: utf-8 -*- +
+ + +
diff --git a/rdfdatabank/templates/doiview.html b/rdfdatabank/templates/doiview.html index 89447d6..ea96f5a 100644 --- a/rdfdatabank/templates/doiview.html +++ b/rdfdatabank/templates/doiview.html @@ -1,43 +1,43 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - DOI view - -

DOI Information for ${c.silo_name}/${c.id}/version${c.version}

-% if c.heading: -

${c.heading}

-%endif -
-% if c.message: -

${c.message}

-% endif -% if c.resp_status: -

Response code from Datacite: ${c.resp_status}

-% endif -% if c.resp_reason: -

Response message from Datacite: ${c.resp_reason}

-% endif -
- -% if c.dois: -

DOIs available for this dataset

-
    - % for v, d in c.dois.iteritems(): -
  • Verison ${v}: ${d}
  • - % endfor -
-% endif - -% if c.version_doi: -

DOI: ${c.version_doi}

-% endif - -% if c.editor: - <%include file="/create_doi.html"/> -% endif - -%if c.metadata: -

Metadata associated with the DOI

-
${c.metadata}
-% endif - +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + DOI view + +

DOI Information for ${c.silo_name}/${c.id}/version${c.version}

+% if c.heading: +

${c.heading}

+%endif +
+% if c.message: +

${c.message}

+% endif +% if c.resp_status: +

Response code from Datacite: ${c.resp_status}

+% endif +% if c.resp_reason: +

Response message from Datacite: ${c.resp_reason}

+% endif +
+ +% if c.dois: +

DOIs available for this dataset

+
    + % for v, d in c.dois.iteritems(): +
  • Verison ${v}: ${d}
  • + % endfor +
+% endif + +% if c.version_doi: +

DOI: ${c.version_doi}

+% endif + +% if c.editor: + <%include file="/create_doi.html"/> +% endif + +%if c.metadata: +

Metadata associated with the DOI

+
${c.metadata}
+% endif + diff --git a/rdfdatabank/templates/embargo_form.html b/rdfdatabank/templates/embargo_form.html index e32d2b2..7b4c55b 100644 --- a/rdfdatabank/templates/embargo_form.html +++ b/rdfdatabank/templates/embargo_form.html @@ -1,27 +1,37 @@ -# -*- coding: utf-8 -*- -
- -% if c.embargos and c.current in c.embargos and c.embargos[c.current]: - % if c.embargos[c.current][0] == True or (isinstance(c.embargos[c.current][0], basestring) and c.embargos[c.current][0].lower() in ['true', '1', 1]): - - - % else: - - - % endif -
- - % if c.embargos[c.current][1]: - - % else: - - % endif -% else: - - -
- - -% endif - -
+# -*- coding: utf-8 -*- +
+% if c.embargos and c.current in c.embargos and c.embargos[c.current]: + % if c.embargos[c.current][0] == True or (isinstance(c.embargos[c.current][0], basestring) and c.embargos[c.current][0].lower() in ['true', '1', 1]): + + Embargoed + Open access +

+ % else: + + Embargoed + Open access +

+ % endif + % if c.embargos[c.current][1]: + + +

+ % else: + + +

+ % endif +% else: + + Embargoed + Open access +

+ +

+

+% endif +
Embargoed: only authorized users of the silo can access the data package
+Open access: all users can access the data package.
+
Embargo date: Date until which a data package should be emargoed. By default, this is set to 70 years from now
+ +
diff --git a/rdfdatabank/templates/file_upload.html b/rdfdatabank/templates/file_upload.html index 89f25d4..3c5324a 100644 --- a/rdfdatabank/templates/file_upload.html +++ b/rdfdatabank/templates/file_upload.html @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- -

Add File to ${c.id}

-

+ - +

+ + +

-

diff --git a/rdfdatabank/templates/files_unpack.html b/rdfdatabank/templates/files_unpack.html index 7e57eb5..9430ea5 100644 --- a/rdfdatabank/templates/files_unpack.html +++ b/rdfdatabank/templates/files_unpack.html @@ -1,18 +1,20 @@ -# -*- coding: utf-8 -*- -% if c.zipfile: -
- -% if len(c.zipfile) > 50: -<% c.disp = "...%s"%c.zipfile[-50:] %> -${c.disp} -% else: -${c.zipfile} -% endif -% if c.newid: -   -% else: -   -%endif - -
-%endif +# -*- coding: utf-8 -*- +% if c.zipfile: +
+ +<% +c.disp = c.zipfile.strip().strip('/').split('/')[-1] +if len(c.disp) > 50: + c.disp = "...%s"%c.disp[-50:] +%> + +% if c.newid: + +% else: + +%endif + +
${c.disp}Id (optional) :Id (optional) : +
+
+%endif diff --git a/rdfdatabank/templates/files_unpack2.html b/rdfdatabank/templates/files_unpack2.html index a3798d5..799de04 100644 --- a/rdfdatabank/templates/files_unpack2.html +++ b/rdfdatabank/templates/files_unpack2.html @@ -1,12 +1,12 @@ -# -*- coding: utf-8 -*- -% if c.zipfile: -
- -% if c.newid: -   -% else: -   -%endif - -
-%endif +# -*- coding: utf-8 -*- +% if c.zipfile: +
+ +% if c.newid: +   +% else: +   +%endif + +
+%endif diff --git a/rdfdatabank/templates/footer.html b/rdfdatabank/templates/footer.html index d3c2e9f..70b9c3c 100644 --- a/rdfdatabank/templates/footer.html +++ b/rdfdatabank/templates/footer.html @@ -1,3 +1,11 @@ -# -*- coding: utf-8 -*- -

Copyright resides with the noted authors/creators of the works held here, unless otherwise noted.

-

Databank API documentation

+# -*- coding: utf-8 -*- +
+ +
+ diff --git a/rdfdatabank/templates/header.html b/rdfdatabank/templates/header.html index 1915c80..0ae526e 100644 --- a/rdfdatabank/templates/header.html +++ b/rdfdatabank/templates/header.html @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- -
- +<% +from pylons import app_globals as ag +%> +

DataBank + %if ag.api_version: + ${ag.api_version} + % endif +

<%include file="/search_form.html"/> -
+ diff --git a/rdfdatabank/templates/home-dev.html b/rdfdatabank/templates/home-dev.html deleted file mode 100644 index 9d9b580..0000000 --- a/rdfdatabank/templates/home-dev.html +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Databank - -

Databank

-

The purpose of DataBank is to provide a pilot version of a robust and efficient system for the safe storage of and open access to (where appropriate) research data. Content could be described as data produced as a result of academic research.. Items might comprise files such as spreadsheets, databases, audio files and images (still and moving). Numerical or other data might be raw data or the data may have been manipulated or processed in some way.

- -

DataBank is not intended to store large-scale data sets such as grid data or other vast data sets. Neither is it intended to replace national, subject or other established data collections. It is envisaged that the future role of DataBank will be to provide a secure and accessible store of small to medium-sized files which have no other option for similar safe storage.

- -

Please note DataBank is a pilot system and still under development.

- -

Users should adhere to the terms and conditions stated for each item.

- -

Browse Data packages

- -
- -

How to use Databank

- -

The admin credentials used in this demo instance of Databank are
username: admin
Password: test

- -

Databank documentation

- - -

Using the Databank API

-

Data can be submitted into Databank by using the Databank API. The API provides a restful web interface to all of Databank's features. -

- - -

Running your own instance of Databank

-

Databank is written in Python. The Databank code is licensed under MIT and can be checked out from https://github.com/dataflow/RDFDatabank -

-

For instructions on installing Databank, follow this document

- - Get in touch with the developers of Databank - -

diff --git a/rdfdatabank/templates/home-prod.html b/rdfdatabank/templates/home-prod.html deleted file mode 100644 index 0c3a0cf..0000000 --- a/rdfdatabank/templates/home-prod.html +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Databank - -

Databank

-

Oxford DataBank is a small collection of non-bibliographic works created by Oxford researchers. The purpose of DataBank is to provide a pilot version of a robust and efficient system for the safe storage of and open access to (where appropriate) Oxford research data. Content could be described as .data produced as a result of academic research.. Items might comprise files such as spreadsheets, databases, audio files and images (still and moving). Numerical or other data might be raw data or the data may have been manipulated or processed in some way.

- -

DataBank is not intended to store large-scale data sets such as grid data or other vast data sets. Neither is it intended to replace national, subject or other established data collections. It is envisaged that the future role of DataBank will be to provide a secure and accessible store of small to medium-sized files which have no other option for similar safe storage. The role of DataBank will be clarified as Oxford University works towards a solution for the storage of and access to its research data. Please note DataBank is a pilot system and still under development.

- -

Users should adhere to the terms and conditions stated for each item.

- - diff --git a/rdfdatabank/templates/home.html b/rdfdatabank/templates/home.html index 9d9b580..976a170 100644 --- a/rdfdatabank/templates/home.html +++ b/rdfdatabank/templates/home.html @@ -1,53 +1,48 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Databank - -

Databank

-

The purpose of DataBank is to provide a pilot version of a robust and efficient system for the safe storage of and open access to (where appropriate) research data. Content could be described as data produced as a result of academic research.. Items might comprise files such as spreadsheets, databases, audio files and images (still and moving). Numerical or other data might be raw data or the data may have been manipulated or processed in some way.

- -

DataBank is not intended to store large-scale data sets such as grid data or other vast data sets. Neither is it intended to replace national, subject or other established data collections. It is envisaged that the future role of DataBank will be to provide a secure and accessible store of small to medium-sized files which have no other option for similar safe storage.

- -

Please note DataBank is a pilot system and still under development.

- -

Users should adhere to the terms and conditions stated for each item.

- -

Browse Data packages

- -
- -

How to use Databank

- -

The admin credentials used in this demo instance of Databank are
username: admin
Password: test

- -

Databank documentation

- - -

Using the Databank API

-

Data can be submitted into Databank by using the Databank API. The API provides a restful web interface to all of Databank's features. -

- - -

Running your own instance of Databank

-

Databank is written in Python. The Databank code is licensed under MIT and can be checked out from https://github.com/dataflow/RDFDatabank -

-

For instructions on installing Databank, follow this document

- - Get in touch with the developers of Databank - -

+# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Databank + +

Welcome to DataBank

+
+

Store, archive, publish and access your data from the web using Databank.

+
+
+

DataBank has a simple restful api and uses a rdf-enhanced pair tree store which can be used by other data services to archive data in Databank, as is used by DataStage

+

DataBank is a pilot system, still under devlopment.

+

We are a part of the Dataflow project

+
+ + + + +

How to use Databank

+ +

The admin credentials used in this demo instance of Databank are
username: admin
Password: test

+ + + + +

Using the Databank API

+ +

Data can be submitted into Databank by using the Databank API. The API provides a restful web interface to all of Databank's features. +

+ + + +

Running your own instance of Databank

+

Databank is written in Python. The Databank code is licensed under MIT and can be checked out from https://github.com/dataflow/RDFDatabank +

+ +

For instructions on installing Databank, follow this document

+ + Get in touch with the developers of Databank + diff --git a/rdfdatabank/templates/item_file_upload.html b/rdfdatabank/templates/item_file_upload.html index 6570d54..748f059 100644 --- a/rdfdatabank/templates/item_file_upload.html +++ b/rdfdatabank/templates/item_file_upload.html @@ -1,9 +1,10 @@ -# -*- coding: utf-8 -*- -

Add File to ${c.id}/${c.path}

-

-

- - - -
-

+# -*- coding: utf-8 -*- +
+ + +

+ + +

+ +
diff --git a/rdfdatabank/templates/items_api.html b/rdfdatabank/templates/items_api.html index 79691ec..7505239 100644 --- a/rdfdatabank/templates/items_api.html +++ b/rdfdatabank/templates/items_api.html @@ -1,160 +1,160 @@ -# -*- coding: utf-8 -*- -
-

Items

-

Sections

-
    - -
  1. /{silo name}/items/{id}
  2. -
  3. /{silo name}/items/{id}/{path}
  4. -
  5. /{silo name}/items/{id}/{path}/{subpath}
  6. -
- - - -
-
-
/{silo name}/items/{id}
- -
-
-

API call to obtain a list of unpack end-points and unpack a zip file into a new / existing dataset

-

Controller: items     action: datasetview

-
-

GET

-
Returns
-
-
401 if not a valid user
-
403 if not authorized
-
404 if dataset id does not exist
-
Accept: text/html
-
Returns an HTML page with list of zipfiles in {id} with form to unpack the zip file
-
Accept: text/plain, application/json
-
200 OK
-
Returns a JSON-encoded list of zipfiles in dataset
-
The file - itemInformationForDataset.txt - contains an example of the data returned (data_returned) -
-
Accept: */*, default
-
Returns text/html - an HTML page with list of zipfiles in {id} with form to unpack the zip file
-
-
-
-

POST Unpack file. Create new dataset/update existing dataset with contents of upacked file

-
Parameters
-
- - - -
filename{name of file to be unpacked, including the path}
Id{(Optional)}. if Id is not supplied, the default dataset id for the new dataset will be {dataset_id}-{file_name}
-
-
Returns
-
-
401 if not a valid user
-
403 if not authorized
-
404 if file to unpack (as given by filename) was not found
-
415 if file is not of type application/zip
-
400 if there was an error reading the zipfile (BadZipFile)
-
Accept:text/html
-
302 to splash page for newly created dataset / updated dataset
-
Accept: text/plain, application/json
-
201/204 on creation/update of dataset
-
Accept: */*, default
-
Returns text/plain - 201/204 on creation/update of dataset
-
-
-
-

PUT, DELETE NOOP

-
- - -
-
-
/{silo name}/items/{id}/{path}
- -
-
-

API call to read the contents of a zip-file (without having to unpack) and unpack a zip file into a new / existing dataset

-

Controller: items     action: itemview

-
-

GET

-
Returns
-
-
401 if not a valid user
-
403 if not authorized
-
404 if path does not exist, or if path does not point to a file
-
415 if file is not of type application/zip
-
400 if there was an error reading the zipfile (BadZipFile)
-
Accept: text/html
-
Returns a HTML page listing the contents of the zipfile
-
Accept: text/plain, application/json
-
200 OK
-
Returns a JSON-encoded hash dict listing the contents of the ziipfile (files / directories in archive), along with the size of the uncompressed member and and the date of last modification of the member as a tuple of 6 values (Year, Month, Day of month, Hours, Minutes, Seconds).
-
The file - itemInformationForZipFileinDataset.txt - contains an example of the data returned (data_returned) -
-
Accept: */*, default
-
Returns text/html - an HTML page listing the contents of the zipfile
-
-
-
-

POST Unpack file. Create new dataset/update existing dataset with contents of upacked file

-
Parameters
-
- - - -
filename{name of file to be unpacked, including the path}
Id{(Optional)}. if Id is not supplied, the default dataset id for the new dataset will be {dataset_id}-{file_name}
-
-
Returns
-
-
401 if not a valid user
-
403 if not authorized
-
404 if path does not exist, or if path does not point to a file
-
415 if file is not of type application/zip
-
400 if there was an error reading the zipfile (BadZipFile)
-
Accept:text/html
-
302 to splash page for newly created dataset / updated dataset
-
Accept: text/plain, application/json
-
201/204 on creation/update of dataset
-
Accept: */*, default
-
Returns text/plain - 201/204 on creation/update of dataset
-
-
-
-

PUT, DELETE NOOP

-
- - -
-
-
/{silo name}/items/{id}/{path}/{subpath}
- -
-
-

API call to retreive a file from the zip file

-

Controller: items     action: subitemview

-
-
TODO:
-
This is yet to be implemented
-
- -
+# -*- coding: utf-8 -*- +
+

Items

+

Sections

+
    + +
  1. /{silo name}/items/{id}
  2. +
  3. /{silo name}/items/{id}/{path}
  4. +
  5. /{silo name}/items/{id}/{path}/{subpath}
  6. +
+ + + +
+
+
/{silo name}/items/{id}
+ +
+
+

API call to obtain a list of unpack end-points, read the contents of a zip-file (without unpacking) and unpack a zip file into a new / existing dataset

+

Controller: items     action: datasetview

+
+

GET

+
Returns
+
+
401 if not a valid user
+
403 if not authorized
+
404 if dataset id does not exist
+
Accept: text/html
+
Returns an HTML page with list of zipfiles in {id} with form to unpack the zip file
+
Accept: text/plain, application/json
+
200 OK
+
Returns a JSON-encoded list of zipfiles in dataset
+
The file + itemInformationForDataset.txt + contains an example of the data returned (data_returned) +
+
Accept: */*, default
+
Returns text/html - an HTML page with list of zipfiles in {id} with form to unpack the zip file
+
+
+
+

POST Unpack file. Create new dataset/update existing dataset with contents of upacked file

+
Parameters
+
+ + + +
filename{name of file to be unpacked, including the path}
Id{(Optional)}. if Id is not supplied, the default dataset id for the new dataset will be {dataset_id}-{file_name}
+
+
Returns
+
+
401 if not a valid user
+
403 if not authorized
+
404 if file to unpack (as given by filename) was not found
+
415 if file is not of type application/zip
+
400 if there was an error reading the zipfile (BadZipFile)
+
Accept:text/html
+
302 to splash page for newly created dataset / updated dataset
+
Accept: text/plain, application/json
+
201/204 on creation/update of dataset
+
Accept: */*, default
+
Returns text/plain - 201/204 on creation/update of dataset
+
+
+
+

PUT, DELETE NOOP

+
+ + +
+
+
/{silo name}/items/{id}/{path}
+ +
+
+

API call to read the contents of a zip-file (without having to unpack) and unpack a zip file into a new / existing dataset

+

Controller: items     action: itemview

+
+

GET

+
Returns
+
+
401 if not a valid user
+
403 if not authorized
+
404 if path does not exist, or if path does not point to a file
+
415 if file is not of type application/zip
+
400 if there was an error reading the zipfile (BadZipFile)
+
Accept: text/html
+
Returns a HTML page listing the contents of the zipfile
+
Accept: text/plain, application/json
+
200 OK
+
Returns a JSON-encoded hash dict listing the contents of the ziipfile (files / directories in archive), along with the size of the uncompressed member and and the date of last modification of the member as a tuple of 6 values (Year, Month, Day of month, Hours, Minutes, Seconds).
+
The file + itemInformationForZipFileinDataset.txt + contains an example of the data returned (data_returned) +
+
Accept: */*, default
+
Returns text/html - an HTML page listing the contents of the zipfile
+
+
+
+

POST Unpack file. Create new dataset/update existing dataset with contents of upacked file

+
Parameters
+
+ + + +
filename{name of file to be unpacked, including the path}
Id{(Optional)}. if Id is not supplied, the default dataset id for the new dataset will be {dataset_id}-{file_name}
+
+
Returns
+
+
401 if not a valid user
+
403 if not authorized
+
404 if path does not exist, or if path does not point to a file
+
415 if file is not of type application/zip
+
400 if there was an error reading the zipfile (BadZipFile)
+
Accept:text/html
+
302 to splash page for newly created dataset / updated dataset
+
Accept: text/plain, application/json
+
201/204 on creation/update of dataset
+
Accept: */*, default
+
Returns text/plain - 201/204 on creation/update of dataset
+
+
+
+

PUT, DELETE NOOP

+
+ + +
+
+
/{silo name}/items/{id}/{path}/{subpath}
+ +
+
+

API call to retreive a file from the zip file

+

Controller: items     action: subitemview

+
+
TODO:
+
This is yet to be implemented
+
+ +
diff --git a/rdfdatabank/templates/itemview.html b/rdfdatabank/templates/itemview.html index 81ed617..0353e64 100644 --- a/rdfdatabank/templates/itemview.html +++ b/rdfdatabank/templates/itemview.html @@ -1,58 +1,123 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Item View - - -

Dataset ${c.id}

- - -% if c.editor: -
- % if c.view == 'editor': - Editor view - % else: - User view - % endif - % if c.version: -
Change view : - Editor | - User

- % else: -
Change view : - Editor | - User

- % endif -
-% endif - -
- -% if c.readme_text: -<%include file="/readme_section.html"/> -% endif - - -% if c.parts: - % if c.show_files: - % if c.editor: - <%include file="/part_list.html"/> - % else: - <%include file="/part_list_display.html"/> - % endif - % else: -

You do not have permission to view these files

- % endif -% else: -

There are no files or folders in this path

-% endif -
- - -% if c.editor and c.view == 'editor': -
- - <%include file="/item_file_upload.html"/> -
-% endif - +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Item View + + + + +

Dataset ${c.id}

+% if c.version: +

Version ${c.version}

+% elif c.versions: +

Version ${c.versions[-1]}

+% endif + + +% if c.editor: +
+

Editorial functions

+ +

+ Upload file

+
+% endif + + +
+% if c.version and c.versions and c.version == c.versions[-1]: +

You are currently viewing the latest version (${c.versions[-1]}) of the data package

+% elif c.version: +

You are currently viewing version ${c.version} of the data package

+% elif c.versions: +

You are currently viewing the latest version (${c.versions[-1]}) of the data package

+% else: +

You are currently viewing the latest version of th data package

+% endif + +% if c.versions: +

View other versions: + % for i in range(1, len(c.versions)-1): + % if c.version and c.versions[i] == c.version: + ${c.versions[i]} | + % else: + ${c.versions[i]} | + % endif + % endfor + % if c.version: + % if c.version == c.versions[-1]: + ${c.versions[-1]} + % else: + ${c.versions[-1]} + % endif + % else: + ${c.versions[-1]} + % endif +

+% endif +
+ +
+ +% if c.parts: +

Data package contents within ${c.path}

+
+ % if c.show_files: + % if c.editor: + <%include file="/part_list.html"/> + % else: + <%include file="/part_list_display.html"/> + % endif + % else: +

You do not have permission to view these files

+ % endif +
+% else: +

There are no files or folders in this path

+% endif +
+ + +% if c.readme_text: +

README text

+
+<%include file="/readme_section.html"/> +
+% endif + + +% if c.editor and c.view == 'editor': + + +% endif + diff --git a/rdfdatabank/templates/itemview_version.html b/rdfdatabank/templates/itemview_version.html index f2f45a7..379f229 100644 --- a/rdfdatabank/templates/itemview_version.html +++ b/rdfdatabank/templates/itemview_version.html @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- -<%inherit file="/base.html" /> -<%def name="head_tags()"> - Item View - -% if c.readme_text: -<%include file="/readme_section.html"/> -% endif -% if c.item: -<%include file="/part_list_display.html"/> -% endif +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Item View + +% if c.readme_text: +<%include file="/readme_section.html"/> +% endif +% if c.item: +<%include file="/part_list_display.html"/> +% endif diff --git a/rdfdatabank/templates/keywords.html b/rdfdatabank/templates/keywords.html new file mode 100644 index 0000000..f13a756 --- /dev/null +++ b/rdfdatabank/templates/keywords.html @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +<%inherit file="/base.html" /> +<%def name="head_tags()"> + Keywords + + +

List of keywords in the site goes here...

diff --git a/rdfdatabank/templates/list_of_datasets.html b/rdfdatabank/templates/list_of_datasets.html index e28b1b1..6c17a94 100644 --- a/rdfdatabank/templates/list_of_datasets.html +++ b/rdfdatabank/templates/list_of_datasets.html @@ -1,22 +1,35 @@ -# -*- coding: utf-8 -*- -% if c.silo_name: -

Datasets:

-
    -<% -n = len(c.items) -%> -

    Number of Datasets: ${n}

    -% for item in sorted(c.items): -
  • - ${item} -
    • Delete dataset
    • -<% -c.current = item -%> -
    • <%include file="/embargo_form.html"/>
    • -
    -
  • -% endfor -
-% endif - +# -*- coding: utf-8 -*- +% if c.silo_name: +

Datasets:

+